<commit_before><?php

namespace AppshedExtension\SpreadsheetBundle\Controller;

use Symfony\Bundle\FrameworkBundle\Controller\Controller;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Template;
use Symfony\Component\HttpFoundation\Response;
use Zend;
use AppShed;
use AppShed\Element\Screen as AppShedScreen;
use AppShed\Element\Item as AppShedItem;
use AppshedExtension\SpreadsheetBundle\Entity\Doc as Doc;

/**
 * @Route("/spreadsheet/write")
 */
class WriteController extends Controller {

    var $filterTypes = array('<', '>', '=', '!=', '<=', '>=',
            //'aroundme'
    );

    private function getTitlesAction($key) {

        $titles = array();

        $worksheet = $this->getDocument($this->getSpreadsheetAdapter(), $key);
        if ($worksheet != null) {

            $lines = $worksheet->getContentsAsRows();
            if (is_array($lines) && isset($lines['0']) && is_array($lines['0'])) {
                $titles = array_keys($lines['0']);
            }
        }
        return $titles;
    }

    /**
     * @Route("/edit")
     * @Template()
     */
    public function indexAction() {
        $request = $this->getRequest();
        $action = '';
        $secret = $request->get('identifier');

        $em = $this->getDoctrine()->getManager();
        $doc = $em->getRepository('AppshedExtensionSpreadsheetBundle:Doc')->findOneBy(array('itemsecret' => $secret));


        if (is_null($doc)) {
            $doc = new Doc();
            $doc->setKey('');
            $doc->setUrl('');
            $doc->setTitles(array());
            $doc->setFilters(array());
            $doc->setItemsecret($secret);
            $doc->setDate(new \DateTime());
        }


        if ($request->isMethod('post')) {

            $url = $request->get('url');
            $key = $this->getKey($url);
            $filters = $request->get('filters', array());
            $titles = array();
            $action = $request->get('action', false);

            $worksheet = $this->getDocument($this->getSpreadsheetAdapter(), $key);
            if ($worksheet != null) {

                $lines = $worksheet->getContentsAsRows();
                if (is_array($lines) && isset($lines['0']) && is_array($lines['0'])) {
                    $titles = array_keys($lines['0']);
                }

                $doc->setUrl($url);
                $doc->setKey($key);
                $doc->setTitles(array_unique($titles));
                $doc->setFilters(array_values($filters));

                $em->persist($doc);
                $em->flush();
            }
        }

        return array(
            'doc' => $doc,
            'action' => $action,
            'filterTypes' => $this->filterTypes
        );
    }

    private function getSpreadsheetAdapter() {

        $client = $this->googleLogin();
        $adapter = new \ZendGData\Spreadsheets($client);
        return $adapter;
    }

    /**
     * @Route("/document/")
     */
    public function documentAction() {
        $request = $this->getRequest();

        if ($request->isMethod('options')) {
            header('Access-Control-Max-Age: 86400');
            header('Access-Control-Allow-Origin: *');
            header('Access-Control-Allow-Methods: GET, POST, OPTIONS');
            header('Access-Control-Allow-Headers: accept, origin, x-requested-with, x-request');
            exit;
        }

        $rowData = $this->getData($_REQUEST['fetchURL']);


        unset($rowData['clientId']);
        unset($rowData['clientSecret']);
        unset($rowData['appId']);
        unset($rowData['appSecret']);
        unset($rowData['itemid']);
        unset($rowData['identifier']);


        $secret = $request->get('identifier');
        $type = $request->get('type', 'normal');

        $em = $this->getDoctrine()->getManager();
        $doc = $em->getRepository('AppshedExtensionSpreadsheetBundle:Doc')->findOneBy(array('itemsecret' => $secret));
        if (!is_null($doc)) {
            $adapter = $this->getSpreadsheetAdapter();
       //     $existingTitles = $doc->getTitles();
            $existingTitles = $this->getTitlesAction($doc->getKey());

            $store = false;
            foreach ($rowData as $titlename => $value) {
                if (strlen($value) == '') {
                    unset($rowData[$titlename]);
                }
                if (!in_array($titlename, $existingTitles)) {
                    $store = true;
                    $this->addTitle($titlename, $adapter, $doc->getKey());
                    $existingTitles[] = $titlename;
                }
            }
<commit_msg>save write doc<commit_after><?php

namespace AppshedExtension\SpreadsheetBundle\Controller;

use Symfony\Bundle\FrameworkBundle\Controller\Controller;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Template;
use Symfony\Component\HttpFoundation\Response;
use Zend;
use AppShed;
use AppShed\Element\Screen as AppShedScreen;
use AppShed\Element\Item as AppShedItem;
use AppshedExtension\SpreadsheetBundle\Entity\Doc as Doc;

/**
 * @Route("/spreadsheet/write")
 */
class WriteController extends Controller {

    var $filterTypes = array('<', '>', '=', '!=', '<=', '>=',
            //'aroundme'
    );

    private function getTitlesAction($key) {

        $titles = array();

        $worksheet = $this->getDocument($this->getSpreadsheetAdapter(), $key);
        if ($worksheet != null) {

            $lines = $worksheet->getContentsAsRows();
            if (is_array($lines) && isset($lines['0']) && is_array($lines['0'])) {
                $titles = array_keys($lines['0']);
            }
        }
        return $titles;
    }

    /**
     * @Route("/edit")
     * @Template()
     */
    public function indexAction() {
        $request = $this->getRequest();
        $action = '';
        $secret = $request->get('identifier');

        $em = $this->getDoctrine()->getManager();
        $doc = $em->getRepository('AppshedExtensionSpreadsheetBundle:Doc')->findOneBy(array('itemsecret' => $secret));


        if (is_null($doc)) {
            $doc = new Doc();
            $doc->setKey('');
            $doc->setUrl('');
            $doc->setTitles(array());
            $doc->setFilters(array());
            $doc->setItemsecret($secret);
            $doc->setDate(new \DateTime());
        }


        if ($request->isMethod('post')) {

            $url = $request->get('url');
            $key = $this->getKey($url);
            $filters = $request->get('filters', array());
            $titles = array();
            $action = $request->get('action', false);

            $worksheet = $this->getDocument($this->getSpreadsheetAdapter(), $key);
            if ($worksheet != null) {

                $lines = $worksheet->getContentsAsRows();
                if (is_array($lines) && isset($lines['0']) && is_array($lines['0'])) {
                    $titles = array_keys($lines['0']);
                }

                $doc->setUrl($url);
                $doc->setKey($key);
                $doc->setTitles(array_unique($titles));
                $doc->setFilters(array_values($filters));

                $em->persist($doc);
                $em->flush();
            }
        }

        return array(
            'doc' => $doc,
            'action' => $action,
            'filterTypes' => $this->filterTypes
        );
    }

    private function getSpreadsheetAdapter() {

        $client = $this->googleLogin();
        $adapter = new \ZendGData\Spreadsheets($client);
        return $adapter;
    }

    /**
     * @Route("/document/")
     */
    public function documentAction() {
        $request = $this->getRequest();

        if ($request->isMethod('options')) {
            header('Access-Control-Max-Age: 86400');
            header('Access-Control-Allow-Origin: *');
            header('Access-Control-Allow-Methods: GET, POST, OPTIONS');
            header('Access-Control-Allow-Headers: accept, origin, x-requested-with, x-request');
            exit;
        }

        $rowData = $this->getData($_REQUEST['fetchURL']);


        unset($rowData['clientId']);
        unset($rowData['clientSecret']);
        unset($rowData['appId']);
        unset($rowData['appSecret']);
        unset($rowData['itemid']);
        unset($rowData['identifier']);


        $secret = $request->get('identifier');
        $type = $request->get('type', 'normal');

        $em = $this->getDoctrine()->getManager();
        $doc = $em->getRepository('AppshedExtensionSpreadsheetBundle:Doc')->findOneBy(array('itemsecret' => $secret));
        if (!is_null($doc)) {

            $adapter = $this->getSpreadsheetAdapter();
//            $existingTitles = $doc->getTitles();
            $existingTitles = $this->getTitlesAction($doc->getKey());

            $store = false;
            foreach ($rowData as $titlename => $value) {
                if (strlen($value) == '') {
                    unset($rowData[$titlename]);
                }
                if (!in_array($titlename, $existingTitles)) {
                    $store = true;
                    $this->addTitle($titlename, $adapter, $doc->getKey());
                    $existingTitles[] = $titlename;
                }
            }

{'fraction_non_alphanumeric': 0.17513397642015005, 'fraction_numerical': 0.002357984994640943, 'mean_word_length': 2.116566466265865, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7590935', 'n_tokens_mistral': 2717, 'n_tokens_neox': 2637, 'n_words': 408}