How to force SSL on Nginx & Varnish

Question: how can i redirect only to https in Nginx and varnish.
I use varnish cache in port 80 and Nginx listening on 8080. Works fine in http, but i need to add SSL. my configuration is as follows
server {
    listen  443 ssl http2 default_server;
    listen  [::]:443 ssl http2;
    server_name afrim.com www.afrim.com;
    port_in_redirect off;

    ssl on;
    ssl_certificate /etc/nginx/ssl/afrim_com_crt.crt;
    ssl_certificate_key /etc/nginx/ssl/afrim_com.key;

    location / {
        proxy_pass http://127.0.0.1:80;
        proxy_set_header Host $http_host;
        proxy_set_header X-Forwarded-Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto https;
        proxy_set_header HTTPS "on";
    }
}

server {
    listen 8080;
    listen [::]:8080;
    server_name afrim.com www.afrim.com;
    root /var/www/html/;
    index index.php;
    port_in_redirect off;

    location / {
        try_files $uri $uri/ /index.php?$args;
    }

    location ~ \.php$ {
        include snippets/fastcgi-php.conf;
        fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
    }

}

server {
    listen  8080;
    listen  [::]:8080;
    server_name afrim.com www.afrim.com;
    return  301 https://afrim.com$request_uri;
}

Answer: As mentioned on https://www.varnish-software.com/developers/tutorials/redirect/ you can use the following VCL code to perform HTTP to HTTPS redirections in Varnish:
vcl 4.1;

import proxy;

backend default {
    .host = "127.0.0.1";
    .port = 8080;
}

sub vcl_recv {
    if ((req.http.X-Forwarded-Proto && req.http.X-Forwarded-Proto != "https") ||
        (req.http.Scheme && req.http.Scheme != "https")) {
        return (synth(750));
    } elseif (!req.http.X-Forwarded-Proto && !req.http.Scheme && !proxy.is_ssl()) {
        return (synth(750));
    }
}

sub vcl_synth {
    if (resp.status == 750) {
        set resp.status = 301;
        set resp.http.location = "https://" + req.http.Host + req.url;
        set resp.reason = "Moved";
        return (deliver);
    }
}

Answer: Try this. These are my configs from an Ubuntu LEMP Digital Ocean Droplet running Nginx, Varnish and PHP-fpm. It also has the redirects to forward all traffic to https://wwww
The files are:

default.vcl
nginx.conf
varnish_daemon
varnish.service

default.vcl
vcl 4.1;

import std;

backend projectname {
    .host = "127.0.0.1";
    .port = "8080";
    .max_connections = 100;
    .connect_timeout        = 5s;
    .first_byte_timeout     = 90s;
    .between_bytes_timeout  = 2s;
}

acl purge {
    "localhost";
    "127.0.0.1";
    "::1";
}

sub vcl_recv {
    if (req.url ~ "^/\.well-known/acme-challenge/") {
        set req.backend_hint = projectname;
        return(pipe);
    }

    # Forward client's IP to the backend
    if (req.restarts == 0) {
        if (req.http.X-Real-IP) {
            set req.http.X-Forwarded-For = req.http.X-Real-IP;
        } else if (req.http.X-Forwarded-For) {
            set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
    unset req.http.proxy;
    set req.url = std.querysort(req.url);
    set req.url = regsub(req.url, "\?$", "");
    set req.http.Surrogate-Capability = "key=ESI/1.0";

    if (std.healthy(req.backend_hint)) {
        set req.grace = 10s;
    }

    if (!req.http.X-Forwarded-Proto) {
        if(std.port(server.ip) == 443) {
            set req.http.X-Forwarded-Proto = "https";
        } else {
            set req.http.X-Forwarded-Proto = "https";
        }
    }

    # === STATIC FILES ===
    # Properly handle different encoding types
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf)$") {
            # No point in compressing these
            unset req.http.Accept-Encoding;
        } elseif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elseif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unknown algorithm (aka crappy browser)
            unset req.http.Accept-Encoding;
        }
    }

    if (req.http.Upgrade ~ "(?i)websocket") {
        return (pipe);
    }

    if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
        set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
        set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
        set req.url = regsub(req.url, "\?&", "?");
        set req.url = regsub(req.url, "\?$", "");
    }

    if (req.method == "PURGE") {
        if (!client.ip ~ purge) {
            return (synth(405, client.ip + " is not allowed to send PURGE requests."));
        }
        return (purge);
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "PATCH" &&
        req.method != "DELETE") {
        return (pipe);
    }

    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|ogg|ogm|opus|otf|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
        unset req.http.Cookie;
        return(hash);
    }

    set req.http.Cookie = regsuball(req.http.Cookie, "(__utm|_ga|_opt)[a-z_]*=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "(__)?hs[a-z_\-]+=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "hubspotutk=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "_hj[a-zA-Z]+=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "(NID|DSID|__gads|GED_PLAYLIST_ACTIVITY|ACLK_DATA|ANID|AID|IDE|TAID|_gcl_[a-z]*|FLC|RUL|PAIDCONTENT|1P_JAR|Conversion|VISITOR_INFO1[a-z_]*)=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");

    if (req.http.cookie ~ "^\s*$") {
        unset req.http.cookie;
    }
}

sub vcl_pipe {
    if (req.backend_hint == projectname) {
        set req.http.Connection = "close";
        return(pipe);
    }
}

sub vcl_hash {
    hash_data(req.http.X-Forwarded-Proto);
}

sub vcl_backend_response {
    if (bereq.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|ogg|ogm|opus|otf|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
        unset beresp.http.Set-Cookie;
        set beresp.ttl = 1d;
    }

    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # Unset the "pragma" header (suggested)
    unset beresp.http.Pragma;

    # Unset the "vary" header (suggested)
    unset beresp.http.Vary;

    set beresp.ttl = 10s;
    set beresp.grace = 2h;
}

nginx.conf
user www-data;
worker_processes auto;
worker_rlimit_nofile 38000;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
    worker_connections 768;
}

http {
    sendfile on;
    autoindex off;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    reset_timedout_connection on;
    keepalive_requests 100000;
    types_hash_max_size 2048;
    server_tokens off;
    server_names_hash_bucket_size 64;
    charset UTF-8;

    # Cloudflare IP's
    set_real_ip_from 220.127.116.11/20;
    set_real_ip_from 22.214.171.124/22;
    set_real_ip_from 18.104.22.168/22;
    set_real_ip_from 126.96.36.199/22;
    set_real_ip_from 184.108.40.206/18;
    set_real_ip_from 22.214.171.124/18;
    set_real_ip_from 184.108.40.206/20;
    set_real_ip_from 184.108.40.206/20;
    set_real_ip_from 188.8.131.52/22;
    set_real_ip_from 22.214.171.124/17;
    set_real_ip_from 22.214.171.124/15;
    set_real_ip_from 126.96.36.199/12;
    set_real_ip_from 126.96.36.199/13;
    set_real_ip_from 188.8.131.52/22;
    set_real_ip_from 2400:cb00::/32;
    set_real_ip_from 2606:4700::/32;
    set_real_ip_from 2803:f800::/32;
    set_real_ip_from 2405:b500::/32;
    set_real_ip_from 2405:8100::/32;
    set_real_ip_from 2a06:98c0::/29;
    set_real_ip_from 2c0f:f248::/32;

    real_ip_header X-Forwarded-For;

    client_max_body_size 128m;

    client_body_buffer_size 2m;
    client_header_buffer_size 64k;
    large_client_header_buffers 16 128k;

    fastcgi_buffers 4 32k;
    fastcgi_buffer_size 32k;
    fastcgi_busy_buffers_size 32k;

    fastcgi_connect_timeout 60s;
    fastcgi_next_upstream_timeout 40s;
    fastcgi_next_upstream_tries 10;
    fastcgi_read_timeout 60s;
    fastcgi_send_timeout 60s;
    fastcgi_cache_lock_timeout 60s;

    open_file_cache max=35000 inactive=30s;
    open_file_cache_valid 30s;
    open_file_cache_min_uses 2;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;

    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;

    gzip on;
    gzip_vary on;
    gzip_proxied any;
    gzip_disable "msie6";

    gzip_min_length 10240;
    #gzip_min_length 1000;
    gzip_buffers 4 16k;
    gzip_comp_level 6;
    gzip_types text/css text/js text/xml text/plain text/javascript
        text/html
        text/x-component
        application/json
        application/xml
        application/x-javascript
        application/javascript
        application/vnd.ms-fontobject
        application/rss+xml
        font/truetype
        font/opentype
        image/svg+xml;

    map_hash_max_size 4200;
    map_hash_bucket_size 4200;

    log_format main
        '$remote_addr - $remote_user [$time_local] "$request" '
        '$status $body_bytes_sent "$http_referer" '
        '"$http_user_agent" "$http_x_forwarded_for"';

    map $http_user_agent $is_non_search_bot {
        default '';
        ~*(google|bing|pingdom|monitis.com|Zend_Http_Clienti) '';
        ~*(http|crawler|spider|bot|search|ForusP|Wget/|Python-urllib|PHPCrawl|bGenius) 'bot';
    }

    map $status $write_log {
        444 0;
        default 1;
    }

    map $http_user_agent $is_bad_client {
        default 0;
        ~*(Sogou) 1;
        ~*(360Spider) 1;
        ~*(aiHitBot|AhrefsBot) 1;
        ~*(betaBot|BlackWidow|Bolt|BLEXBot|BUbiNG) 1;
        ~*(CazoodleBot|CPython|CCBot|ChinaClaw|Curious|CRAZYWEBCRAWLER|Custo) 1;
        ~*(Default|DIIbot|DISCo|discobot) 1;
        ~*(Exabot|eCatch|ecxi|EirGrabber|EmailCollector|EmailSiphon|EmailWolf|ExtractorPro|EyeNetIE) 1;
        ~*(FlashGet|Findxbot) 1;
        ~*(GetRight|GetWeb!|Go!Zilla|Go-Ahead-Got-It|Go.*package.*|GrabNet|Grafula|GT::WWW|GuzzleHttp) 1;
        ~*(heritrix|HaosouSpider|HMView|HTTP::Lite|HTTrack) 1;
        ~*(ia_archiver|IDBot|id-search|id-search.org|InterGET|InternetSeer.com|IRLbot) 1;
        ~*(JetCar) 1;
        ~*(larbin|LeechFTP|Lightspeedsystems|litemage_walker|LinksManager.com_bot|Lipperhey|linkwalker|LinkpadBot|lwp-trivial|ltx71) 1;
        ~*(Maxthon$|Mail.RU_Bot|MegaIndex.ru|meanpathbot|MFC_Tear_Sample|microsoft.url|Microsoft-IIS|Mozilla.*Indy|Mozilla.*NEWT|MJ12bot|MSFrontPage) 1;
        ~*(Navroad|NearSite|NetAnts|NetLyzer.*FastProbe|NetSpider|NetZIP|Nutch) 1;
        ~*(Octopus) 1;
        ~*(PageGrabber|panscient.com|pavuk|PECL::HTTP|PeoplePal|pcBrowser|Pi-Monster|PHPCrawl|PleaseCrawl|psbot|prijsbest|python-requests) 1;
        ~*(Qwantify) 1;
        ~*(RealDownload|ReGet|RedesScrapy|Rippers|RocketCrawler) 1;
        ~*(SBIder|Scrapy|Screaming|ScreenerBot|SEOprofiler|SeaMonkey$|SeznamBot|SemrushBot|sitecheck.internetseer.com|SiteSnagger) 1;
        ~*(SmartDownload|Snoopy|SputnikBot|Steeler|SuperBot|SuperHTTP|Surfbot|sqlmap) 1;
        ~*(tAkeOut|Teleport|Toata|TwengaBot|Typhoeus) 1;
        ~*(URI::Fetch|User-Agent|UserAgent) 1;
        ~*(voltron|Vagabondo|VoidEYE|Visbot) 1;
        ~*(webalta|WebAuto|[Ww]eb[Bb]andit|WebCollage|WebCopier|WebFetch|WebLeacher|WebReaper|WebSauger|WebStripper|WebWhacker|WhatsApp) 1;
        ~*(WebZIP|Wget|Widow|Wotbox|WWW-Mechanize|WWWOFFLE) 1;
        ~*(zermelo|Zeus|Zeus.*Webster|ZyBorg) 1;
    }

    map $uri $redirected_url {
        default "none";
    }

    limit_req_zone $binary_remote_addr zone=one:10m rate=30r/s;
    limit_req_zone $is_non_search_bot zone=bots:10m rate=22r/s;

    limit_req_log_level error;

    index index.php index.html;

    upstream fastcgi_backend {
        server unix:/var/run/php/php5.6-fpm.sock;
    }

    proxy_cache_path /var/cache/varnish levels=1:2 keys_zone=my_cache:8m max_size=1g inactive=60m use_temp_path=off;

    server {
        listen 8080;
        listen [::]:8080;
        server_name example.com;
        # redirect http to https://wwww
        return 301 https://www.example.com$request_uri;
    }
    server {
        listen 8080;
        listen [::]:8080;
        root /var/www/current/default_site;

        server_name www.example.com;

        location / {
            try_files $uri $uri/ /index.php?q=$uri&$args;
        }

        error_page 404 /404.html;
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
            root /usr/share/nginx/html;
        }

        location ~ \.php$ {
            include snippets/fastcgi-php.conf;
            fastcgi_pass unix:/run/php/php5.6-fpm.sock;
        }
    }

    server {
        listen 443;
        listen [::]:443;
        server_name example.com;
        ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; # managed by Certbot
        ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; # managed by Certbot
        include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
        ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
        # redirect http to https://wwww
        return 301 https://www.example.com$request_uri;
    }

    server {
        listen [::]:443 ssl http2;
        listen 443 ssl;
        root /var/www/current/default_site;
        server_name www.example.com;
        ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; # managed by Certbot
        ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; # managed by Certbot
        include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
        ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

        ssl_session_cache shared:SSL:10m;
        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
        add_header "X-XSS-Protection" "1; mode=block";

        if ($request_uri ~* "^(.*/)index\.php(/?)(.*)" ) {
            return 301 $1$3;
        }

        if ($is_bad_client) {
            return 444;
        }

        location ~* \.(ico|jpg|jpeg|png|gif|svg)$ {
            access_log off;
            add_header Cache-Control "public";
            add_header X-Frame-Options SAMEORIGIN always;
            expires 365d;
        }

        location ~* \.(js|css|pdf)$ {
            expires 365d;
            access_log off;
            add_header Cache-Control "public";
            add_header X-Frame-Options SAMEORIGIN always;
            add_header Access-Control-Allow-Origin "https://fast.fonts.net";
        }

        location ~* \.(html|swf)$ {
            expires 30d;
            add_header Cache-Control "public";
            add_header X-Frame-Options SAMEORIGIN always;
        }

        location ~* \.(eot|ttf|otf|woff|woff2)$ {
            access_log off;
            add_header Cache-Control "public, no-transform";
            add_header X-Frame-Options SAMEORIGIN always;
            expires 365d;
        }

        location = /favicon.ico {
            log_not_found off;
            access_log off;
        }

        location = /robots.txt {
            allow all;
            log_not_found off;
            access_log off;
        }

        location / {
            proxy_cache my_cache;
            proxy_cache_revalidate on;
            proxy_cache_min_uses 3;
            proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
            proxy_cache_background_update on;
            proxy_cache_lock on;

            proxy_ignore_headers X-Accel-Expires;
            proxy_ignore_headers Expires;
            proxy_ignore_headers Cache-Control;

            proxy_pass http://127.0.0.1:80;
            # ALSO TRY YOUR SERVER IP proxy_pass http://18.104.22.168:80;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto https;
            proxy_set_header X-Forwarded-Port 443;
            proxy_set_header Host $host;
        }
        error_page 404 /404.html;
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
            root /usr/share/nginx/html;
        }

        location ~ \.php$ {
            include snippets/fastcgi-php.conf;
            fastcgi_pass unix:/run/php/php5.6-fpm.sock;
        }

    }
}

varnish_daemon
This is the code from "nano /etc/default/varnish"
DAEMON_OPTS="-a :80 \
    -T localhost:6082 \
    -p feature=+http2 \
    -f /etc/varnish/default.vcl \
    -S /etc/varnish/secret \
    -s malloc,1g"

varnish.service
This is the code from "nano /lib/systemd/system/varnish.service"
[Unit]
Description=Varnish HTTP accelerator
Documentation=https://www.varnish-cache.org/docs/6.1/ man:varnishd

[Service]
Type=simple
LimitNOFILE=131072
LimitMEMLOCK=82000
ExecStart=/usr/sbin/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -p feature=+http2 -s malloc,1G
ExecReload=/usr/share/varnish/varnishreload
ProtectSystem=full
ProtectHome=true
PrivateTmp=true
PrivateDevices=true

[Install]
WantedBy=multi-user.target

Comment: As it's currently written, your answer is unclear. Please [edit] to add additional details that will help others understand how this addresses the question asked. You can find more information on how to write good answers [in the help center](/help/how-to-answer).

How to force SSL on Nginx & Varnish


ERRO urllib.error.HTTPError: HTTP Error 404: Not Found

Question: Tenho o seguinte código, simples.
from urllib.request import urlopen
from bs4 import BeautifulSoup

word_site = urlopen('https://svnweb.freebsd.org/csrg/share/dict/words?view=markup&pathrev=61569')
bs = BeautifulSoup(word_site, parser='html.parser')

print(bs.h1)

Já procurei por todos os lugares e não encontrei resposta satisfatória. Esse código gera o seguinte erro:
Traceback (most recent call last):
  File "<input>", line 1, in <module>
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2019.3.1\plugins\python-ce\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2019.3.1\plugins\python-ce\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/joaof/OneDrive/Programacao/Cursos/Cursos/curso-de-programacao-em-python-do-basico-ao-avancado/exercicios/secao08/ex60.py", line 5, in <module>
    word_site = urlopen('https://svnweb.freebsd.org/csrg/share/dict/words?view=markup&pathrev=61569')
  File "C:\Users\joaof\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 214, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\joaof\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 523, in open
    response = meth(req, response)
  File "C:\Users\joaof\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 632, in http_response
    response = self.parent.error(
  File "C:\Users\joaof\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 561, in error
    return self._call_chain(*args)
  File "C:\Users\joaof\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 494, in _call_chain
    result = func(*args)
  File "C:\Users\joaof\AppData\Local\Programs\Python\Python39\lib\urllib\request.py", line 641, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found

Por favor, alguém consegue me ajudar?
PS: a URL existe e está funcionando normalmente.
Answer: Geralmente não precisa mas essa página que você está trabalhando precisa de um agente, usei a identificação sendo o navegador firefox mozilla.
import requests, bs4
url = "https://svnweb.freebsd.org/csrg/share/dict/words?view=markup&pathrev=61569"
headers = {"User-Agent":"Mozilla/5.0"}
response = requests.get(url, headers=headers)
soup = bs4.BeautifulSoup(response.text, 'html.parser')

'''
# Imprimir tudo
print(soup.text)
'''

# Imprimir linha por linha, removendo linhas sem dado
for x in soup.text.splitlines():
    if len(x) > 0:
        print(x)

Comment: Muito obrigado Gilmar, baseado na sua explicação e solução, refiz meu código.
Comment: opa, legal, de nada.


Proof of well-ordering of $\mathbb{N}$ using modified induction and contraposition.

Question: This question regards an alternative proof of the theorem of well-ordering
of $\mathbb{N},$ using a modified principle of induction and contraposition,
found on page 101 of Fundamentals of Mathematics, Volume 1 Foundations of Mathematics: The Real Number System and Algebra; Edited by H. Behnke, F. Bachmann, K. Fladt, W. Suess and H. Kunle.
The square brackets following quantifiers indicate the scope of the bound variables.  The way I read my notation is, for example:
$$
\forall_{n<m}\left[n\in\mathcal{M}\right]
$$
For all $n$ less than $m$, $n$ is an element of $\mathcal{M}$.
The question: Is the following proof correct?
The modified principle of induction is: If the number $m$ is contained
in the number set $\mathcal{M}$ whenever $n\in\mathcal{M}$ for all
numbers $n<m$, then $\mathcal{M}=\mathbb{N}.$
The theorem of well-ordering of the natural numbers is: Every nonempty set of natural numbers contains a smallest number.
This is what I am trying to accomplish:

As another method of proving the same theorem, we note that, if we
  replace $\mathcal{M}$ by the set $\mathbb{N}-\mathcal{M}$ of the
  numbers not in $\mathcal{M},$ our modified principle of induction
  can be transformed, by contraposition and other purely logical operations,
  into the desired theorem of well-ordering of the natural numbers.

One form of the contrapositive of implication is:
$$
\left(p\implies q\iff\lnot q\implies\lnot p\right).
$$
The modified principle of induction is formalized as
$$
\forall_{m}\left[\forall_{n<m}\left[n\in\mathcal{M}\right]\Rightarrow m\in\mathcal{M}\right]\Rightarrow\mathcal{M}=\mathbb{N}.
$$
Rename $\mathcal{M}\mapsto\overline{\mathcal{M}}$ and write the result
as the contrapositive of the original expression, followed by some
transformations
$$
\overline{\mathcal{M}}\neq\mathbb{N}\Rightarrow\lnot\left(\forall_{m}\left[\forall_{n<m}\left[n\in\mathcal{M}\right]\Rightarrow m\in\mathcal{M}\right]\right)
$$
$$
\iff\overline{\mathcal{M}}\subset\mathbb{N}\Rightarrow\exists_{m}\left[\neg\left(\forall_{n<m}\left[n\in\overline{\mathcal{M}}\right]\Rightarrow m\in\overline{\mathcal{M}}\right)\right]
$$
$$
\iff\overline{\mathcal{M}}\subset\mathbb{N}\Rightarrow\exists_{m}\left[\forall_{n<m}\left[n\in\overline{\mathcal{M}}\right]\land m\notin\overline{\mathcal{M}}\right].
$$
Now define $\overline{\mathcal{M}}\equiv\mathbb{N}-\mathcal{M},$
and replace $\overline{\mathcal{M}}\mapsto\mathcal{M}$ according
to that meaning
$$
\emptyset\ne\mathcal{M}\subseteq\mathbb{N}\Rightarrow\exists_{m}\left[\forall_{n<m}\left[n\notin\mathcal{M}\right]\land m\in\mathcal{M}\right].
$$
This statement reads: That $\mathcal{M}$ is a non-empty subset of
$\mathbb{N}$ implies the existence of $m$ such that for all $n<m$
we have $n\notin\mathcal{M}$ and $m\in\mathcal{M}.$ That is $m$
is the smallest element of $m.$
Comment: Yes, it's correct.
Answer: Apparently others affirm the validity of my proof, and I believe it to be formally correct, so my answer is: Yes the proof is correct.
For anybody who uses this proof, I urge him or her to explore its meaning by considering special cases such as $\mathcal{M}=\left\{n\backepsilon n>m\right\},$ etc.


Intercepting keyboard and mouse events from focused applications on OS X

Question: Soon I will have to work with OS X and tools like hammerspoon are missing some important capabilities for me. I need to be able to intercept keyboard and mouse events completely from the focused application. Say I ctrl+alt+apple+left_click on an application, I don't want the application to know about that left click. So far the only thing I came up with was to build a transparent fullscreen application, though I'm not sure how feasible that is yet.
Any better idea or hints how to go about this in a language of your choice?
Thanks!
Answer: You will need to create an event tap.  However, the application will have to run as the root user, or the user will have to authorize that the application has been granted rights to accessibility features.
Apple's documentation can be found here.
Interestingly enough, I am in the process of writing a blog post about how to use event taps (including an ObjectiveC API that I wrote for my own use), but the post won't be made available for another week or so.
Comment: Thanks! :) Please add another comment with the link so I can check it out!


linux log memory hogging issue

Question: We have a VPS server (it's using Virtuozzo). On a few occasions now, our VPS memory was fully used up and no new connections could be made to the server on SSH, SMTP, or POP. The only thing that works is connecting to the web service. Luckily, plesk is running on the VPS and we have been able to reboot it through plesk (as well as see that the RAM is 100% used).
I would like to find what process is causing this. I have a feeling it's MySQL, but don't really know. Is there some sort of logging I could implement that would help me find out what was the cause of this next time it happens?
Thanks.
Comment: This doesn't smell like a memory leak to me (though I could be wrong) - Does Virtuozzo have a "system console" access? If so, can you get in to the system through that & try to troubleshoot from there?
Answer: In situations such as yours I have run ps aux from cron every minute so I could look back and find the memory hog.  Put the following in your crontab:
* * * * * (date ; ps aux) >> $HOME/ps-aux

Caution: this may build a large log file over time.
After your next crash, you can look back through the file looking for processes that use a lot of memory (the "%MEM", "VSZ", and "RSS" columns would be where to look).
Comment: Thanks. I was hoping to avoid something like that. I guess there might not be another option.
Answer: Have you tried anything like atop?

linux log memory hogging issue


Como funciona o Tracking / Chache de objetos do EntityFramework?

Question: Sempre que utilizei o EntityFramework em um projeto, para adicionar um registro no banco de dados por meio dele, de uma classe que contenha membros de tipo de classes (geralmente por conta de ForeignKeys), eu fazia o seguinte:

Após a instância da entidade, preenchia as propriedades de tipos primitivos;
Setava as propriedades de tipos de classes como null.

E assim para mim sempre funcionou, até que em um caso específico como o que apresentei nessa questão: EntityFramework reclamando de duplicidade de Identificador mesmo com a propriedade estando nula.
Há algumas questões aqui que tratam dessa questão, sempre casos isolados e uma forma de contornar.

Como, afinal de contas, funciona o Tracking de objetos do EntityFramework?
Ao trabalhar com ASP.NET MVC (projetos web) que funciona de forma a não manter informações em memória. Ou seja, o contexto do EntityFramework e todas as outras coisas são instanciadas quando uma requisição é feita e destruídas quando ela termina de retornar seu resultado. Logo, sempre que efetuo um POST para uma Action, que fará o bind de uma classe conforme parâmetro da Action, sei que tenho uma instância não monitorada pelo EntityFramework, certo? Por que então um simples:

    if (ModelState.IsValid)
    {
        _context.Add(model);
        _context.SaveChanges();
    }

Funciona?

Como no caso da pergunta que referenciei, porque houve problemas de monitoramento?
Qual seria a forma mais "completa" para lidar com os Inserts e Updates e não ter problemas com o Tracking de objetos?
Habilitar ou Desabilitar o LazyLoading e criação de proxies e o monitoramento de alterações (AutoDetectChangesEnabled) afeta como o Tracking?
Answer: São muitas perguntas pra responder, e um pouco complicadas, mas vou tentar
1 - O ChangeTracking do EntityFramework passa a monitorar uma instância de uma entidade após ela ser "attachada" (anexada) a uma instância de um DbContext.
2 - Quando você chama um método de um DbSet, por exemplo, Add(T model), o EntityFramework vai attachar o objeto ao DbContext e marcar o estado dele como Added.
Lembrando que você não pode anexar um mesmo objeto a duas instâncias distintas de DbContext
Você pode verificar qual o estado atual de um objeto através do método do Entry(object instance).
3 - O EntityFramework utiliza o Identity Map Pattern. Com isso, uma instância de DbContext só pode possuir uma única instância de um objeto com a mesma chave (Primary Key) anexada ao ChangeTracker. Por exemplo, se você carregou o registro com o Id = 1 de uma entidade Pessoa, você não pode anexar outro objeto com o mesmo Id = 1, senão você terá problemas.
4 - A melhor forma de tratar inserts e updates é isolar as operações com entidades de forma similar a uma transação. Uma instância de um DbContext deve ter uma vida curta, ou seja, você instancia o DbContext, faz o que precisa e logo em seguida descarta a instância. Preciso melhorar a resposta para esta questão 4, mas vou adiantando que  existe um NuGet package muito bom pra tratar updates mais complexos, chamado GraphDiff.
5 - Eu não utilizo LazyLoading, nem recomendo, prefiro ter controle total sobre o que quero carregar do meu banco de dados e quando necessário carrego de forma prematura os dados que vou precisar, utilizando Include(). Então, por padrão eu desabilito. O Lazy Loading requer a criação de classes proxy, e requer que as propriedades da Entidade sejam virtuais. Quanto ao AutoDetectChanges, caso seja desabilitado, você terá que chamar o método DetectChanges() manualmente pra que o EntityFramework descubra quais as propriedades sofreram modificações na hora de gerar SQL que atualizam entidades.
Comment: Na verdade, não é uma boa prática ficar passando uma instância de um contexto como parâmetro em várias camadas de sua aplicação (controller, serviço, repositório, etc.). Crie o seu DbContext só na implementação do seu Repositório. UnitOfWork + Repositório + EntityFramework não recomendo porque nunca se sabe o que foi feito em um DbContext nas outras camadas, você se perde muito fácil. http://rob.conery.io/2014/03/04/repositories-and-unitofwork-are-not-a-good-idea/
Comment: +1 pelo GraphDiff. Sensacional o pacote.
Comment: Infelizmente não consigo mais interagir na outra questão. Também não consigo votar aqui. Apenas marcar a reposta como aceita. Bom, na outra pergunta, sobre o método `AtualizarPrecoProdutoAsync`, verdadeiramente obtenho o uma instância do produto ali para então alterar a propriedade `Preco` e então gravar a alteração. **Porém**, o erro acontece no método `AddAsync`, nesse momento ainda não foi executado o `AtualizarPrecoProdutoAsync`.
Comment: Sobre o `AtualizarPrecoProdutoAsync`, reitero, é o que você disse sim, exatamente isso! Me atrapalhei no debug com os `async's` da vida. Muito obrigado por sua explicação.
Answer: Como, afinal de contas, o funciona o Tracking de objetos do EntityFramework?
Primeiro é importante explicar que um contexto existe durante o ciclo de vida de um Controller, no caso específico de você estar trabalhando usando o MVC. Os valores carregados pelo Controller só são expirados quando o Controller em questão sofre um Dispose, fazendo o contexto também sofrer um Dispose.
Ao carregar um objeto (em inglês chamam de entry, e eu vou chamar de registro), o registro é carregado com os valores originais do banco de dados, e eles ficam lá mesmo que você altere o registro. Isto faz com que, ao modificar um registro, o Entity Framework saiba que este registro já foi carregado previamente e saiba o que deve alterar.
O registro é localizado normalmente pela chave dele. Para entender isto melhor, é preciso entender a implementação do DbSet<T>. Ele é uma coleção observável (ObservableCollection). Ao chamar SaveChanges, o que está sendo feito é:

Conferir se alguma coleção está marcada como alterada;
Para cada registro que foi marcado como alterado, o Entity Framework executa alguma operação lógica para inserir, atualizar ou excluir um registro no banco de dados. A instrução executada está ligada a EntityState, um Enum usado para identificar o estado do registro dentro do Entity Framework.

Logo, sempre que efetuo um POST para uma Action, que fará o bind de uma classe conforme parâmetro da Action, sei que tenho uma instância não monitorada pelo EntityFramework, certo?
Errado. Se o registro já foi carregado previamente (num Edit, por exemplo), ele já está sendo "observado". Ao aplicar um POST para salvar a edição, o objeto vindo da View é sincronizado da seguinte forma:
context.Entry(obj).State = EntityState.Modified;

Repare que o contexto aqui é um contexto novo, sem a informação da requisição anterior. Quando digo "o registro já foi carregado previamente", digo que ele foi carregado em tela, e não em Controller. Ao realizar o POST, você está enviando informações e o contexto trabalha de forma otimista, ou seja, o registro é novamente carregado em Entry() caso já não tenha sido carregado.
Nesta linha são feitas duas coisas:

Localizar um registro com base em outro;
Atualizar os valores do registro observado com base nos registros em tela.

O caso do Add é mais simples. Você está dizendo ao contexto que o objeto não existe, então o contexto considera o objeto como novo.
Como no caso da pergunta que referenciei, porque houve problemas de monitoramento?
Vou responder na pergunta.
Qual seria a forma mais "completa" para lidar com os Inserts e Updates e não ter problemas com o Tracking de objetos?

Não implementando repositórios (o Entity Framework já implementa repositórios);
Não implementando camada de Serviço (Controller já é uma camada de serviços, diferente, mas é);
Não instanciando dois contextos desnecessariamente;
Não selecionando objetos em um contexto e tentando salvar no outro;
Usando o código mais simples possível, conforme (quase) todos os exemplos ilustrados aqui e em outros sites especializados.

Habilitar ou Desabilitar o LazyLoading e criação de proxies e o monitoramento de alterações (AutoDetectChangesEnabled) afeta como o Tracking?
Carga preguiçosa e monitoramento são coisas independentes. Uma não afeta a outra e vice-versa.
AutoDetectChangesEnabled faz o tracking não funcionar. Toda e qualquer alteração tem que ser invocada por métodos manuais.
Comment: Vale ressaltar que a resposta que você deu assume que está sendo usado um controller do template do Visual Studio que adiciona um DbContext no início da requisição e chama um dispose ao final. O que não é sempre o caso. Se eu estiver utilizando uma aplicação WPF ou Windows Forms, por exemplo, não há Controllers, ou seja, o ciclo de vida de um Contexto não está relacionado a um Controller. Mas gostei da sua didática para explicar, tenho que melhorar isso nas minhas respostas.
Comment: Que explicação excelente, obrigado! Mas não entendi o ciclo de vida do conroller. Eu executo um `Edit` e obtenho um registro (pensava que ao obter minha página carregada no browser o controller, e claro o contexto, já haveria sido destruído no servidor), e ao efetuar o **POST** que tudo já teria se perdido e com uma nova instância do controller e do contexto não haveria registros já em observação pelo entityFramework. Então o Controller só é destruído quando eu, em minha sessão, mudo de um controller para outro pela navegação?
Comment: `um contexto existe durante o ciclo de vida de um Controller`. Isto é verdade apenas se você fizer o que está certo. Ou seja se o contexto não for um campo de classe mas sim de instância.
Comment: Na outra questão o que eu faço é carregar os produtos para para criar um `DropDownList` com os produtos, porém utilizo o `AsNoTracking()`. Já o `ProdutoId` vêm da View, o value selecionado no `DropDownList`.
Comment: Pelo o que entendi aqui **[what-is-the-lifetime-of-a-asp-net-mvc-controller](http://stackoverflow.com/questions/2355139/what-is-the-lifetime-of-a-asp-net-mvc-controller)**, o controller (e consequentemente o DbContext criado para ele, são criados a cada requisição e destruídos quando a requisição acontece. Então um ***GET*** ao ***Action Edit*** e depois um ***POST*** ao ***Action*** `[HttpPost] Edit(Model model)` é referente à criar e destruir  o controller (e o DbContext) duas vezes. Enfim, cada vez estou sem entender mais... rs =(
Comment: No comentário acima eu ressaltei que o ciclo de vida do DbContext não tem nada em haver com o ciclo de vida de um Controller. O que acontece é que como padrão geralmente um DbContext é instanciado com o Controller.
Comment: @Severo É isso mesmo. A cada requisição, o *Controller* destrói o `DbContext` ao final da requisição. Uma parte da resposta nao deixou isso claro, então vou editar.
Comment: @JonePolvora É que no caso dele, ele deixa claro que está usando ASP.NET MVC, não WPF, mas o princípio é o mesmo. Sobre a sua resposta, ela é complementar à minha e poderia muito bem ser a resposta aceita. Apenas gostaria que você aprendesse a sintaxe Markdown. Ela é bastante simples e pode ajudar um montão na hora de formatar uma resposta. Os links para ajuda com o Markdown ficam junto do painel resposta.
Comment: @Cigano eu conheço markdown, foi preguiça mesmo rsrs. Vou começar a melhorar a formatação e a didática nas respostas. Valew.
Comment: @BrunoCosta Normalmente é assim, certo? Outra coisa. Ainda que seja de classe, o `Dispose` descarta o contexto ao final da execução da requisição, com o descarte do *Controller*.
Comment: Relacionado: http://pt.stackoverflow.com/questions/51536/quando-usar-entity-framework-com-repository-pattern/80696#80696


Média não está sendo armazenada-C

Question: Boa noite!
Ao rodar o programa e informar os dados Nome, Nota1 e Nota2, é esperado que o programa crie um arquivo chamado "turma.txt" com o nome do aluno, notas e a média.
Contudo meu problema está em ler a média corretamente, ao abrir o txt, a média está sempre igual a 0.00. Queria saber como corrigir esse erro.
Obrigado.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

struct
{
    char nome[50];
    float nota1, nota2,media;

}aluno [10];

int main()
{
FILE *arq;
int j;
float media;

    for(j=0; j<10; j++)
    {
        printf("\n \t Dados do aluno %d", j+1);

        printf("\n\n Nome: ");
        scanf("%s", aluno[j].nome);
        fflush(stdin);

        printf("\n Nota 1: ");
        scanf ("%f", &aluno[j].nota1);

        printf("\n Nota 2: ");
        scanf ("%f", &aluno[j].nota2);
    }

    for (j=0; j<10; j++);
    {
        aluno[j].media=((aluno[j].nota1)+(aluno[j].nota2))/2;

    }
    printf("\n\n");
    printf("\t Calculando medias e criando arquivo.....");
    printf("\n");

    arq = fopen("turma.txt","w");

    for (j=0; j<10; j++)
    {

        fprintf(arq,"\n \t Dados do aluno %d \n", j+1);

        fprintf(arq,"\n\n Nome: %s, ",aluno[j].nome);

        fprintf(arq,"%.2f, ", aluno[j].nota1);

        fprintf(arq,"%.2f; ", aluno[j].nota2);

        fprintf(arq,"%.2f \n", aluno[j].media);

    }
    fclose(arq);
    system("cls");
    system("pause");

}

Comment: Você já verificou se seu array `aluno` está sendo carregado corretamente? Os nomes dos alunos tem sempre uma única palavra, isto é, sem espaços, como por ex.: `José Roberto`?
Answer: Gustavo, no seu código você está iterando 2 vezes desnecessariamente.
Eu tirei o loop que calcula a média e coloquei dentro do for que faz as leituras das notas.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

struct {
    char nome[50];
    float nota1, nota2, media;
} aluno[10];

int main() {

    FILE *arq;
    int j;
    float media;
    int total_aluno = 2;

    for(j=0; j<total_aluno; j++) {
        printf("\n \t Dados do aluno %d", j+1);

        printf("\n\n Nome: ");
        scanf("%s", &aluno[j].nome);
        fflush(stdin);

        printf("\n Nota 1: ");
        scanf ("%f", &aluno[j].nota1);

        printf("\n Nota 2: ");
        scanf ("%f", &aluno[j].nota2);
        // Calculando as médias aqui
        aluno[j].media = (aluno[j].nota1 + aluno[j].nota2)/2;
    }

    printf("\n\n");
    printf("\t Calculando medias e criando arquivo.....");
    printf("\n");

    arq = fopen("turma.txt","w");

    for (j=0; j<total_aluno; j++) {

        fprintf(arq,"\n \t Dados do aluno %d \n", j+1);

        fprintf(arq,"\n\n Nome: %s, ",aluno[j].nome);

        fprintf(arq,"%.2f, ", aluno[j].nota1);

        fprintf(arq,"%.2f; ", aluno[j].nota2);

        fprintf(arq,"%.2f \n", aluno[j].media);

    }
    fclose(arq);
    system("cls");
    system("pause");
}

Eu achei bem estranho o erro, mas me parece algo com a referência da variável.
Answer: Antes do fclose(arq); chame o fflush(arq);
Comment: Mesmo assim, a média continua aparecendo zerada. O estranho é estar armazenando o valor 0.00 para ela.

Média não está sendo armazenada-C


Recuperar valores de um array da linha seguinte na linha anterior

Question: Tenho um array no formato abaixo:
Array
(
<code>[0] => Array
    (
        [A] => 000001001000003001
        [B] => VLR REF COMPENSACAO DE VENDA CONFORME CO1
        [C] => 1.1.2.01.020001
        [D] => Z4
        [E] =>
        [F] =>
        [G] =>
        [H] =>           100.000,00
        [I] =>
        [J] =>       4.703.615,60 C
    )

[1] => Array
    (
        [A] =>
        [B] => MPRA DE EQUIPAMENTO DO FORNECEDOR PARANA 1
        [C] =>
        [D] =>
        [E] =>
        [F] =>
        [G] =>
        [H] =>
        [I] =>
        [J] =>
    )

[2] => Array
    (
        [A] =>
        [B] => EQUIPAMENTOS S.A. - NF VENDA 000000261 1
        [C] =>
        [D] =>
        [E] =>
        [F] =>
        [G] =>
        [H] =>
        [I] =>
        [J] =>
    )
[3] => Array
    (
        [A] => 008850001000002098
        [B] => PG. DOC. 3 000113169  TP:NF FORNEC.:0000
        [C] =>
        [D] => Y1
        [E] =>
        [F] =>
        [G] =>
        [H] =>             2.188,35
        [I] =>
        [J] =>           8.901,01 C
    )

[4] => Array
    (
        [A] =>
        [B] => 38-PARANA EQUIPAMENTOS S.A.
        [C] =>
        [D] =>
        [E] =>
        [F] =>
        [G] =>
        [H] =>
        [I] =>
        [J] =>
    )
</code>
)
E preciso que o array fique em outro formato, conforme exemplo abaixo:
Array
(
<code>[0] => Array
    (
        [A] => 000001001000003001
        [B] => VLR REF COMPENSACAO DE VENDA CONFORME CO1 MPRA DE EQUIPAMENTO DO FORNECEDOR PARANA 1 EQUIPAMENTOS S.A. - NF VENDA 000000261 1
        [C] => 1.1.2.01.020001
        [D] => Z4
        [E] =>
        [F] =>
        [G] =>
        [H] =>           100.000,00
        [I] =>
        [J] =>       4.703.615,60 C
    )

[1] => Array
    (
        [A] =>
        [B] =>
        [C] =>
        [D] =>
        [E] =>
        [F] =>
        [G] =>
        [H] =>
        [I] =>
        [J] =>
    )

[2] => Array
    (
        [A] =>
        [B] =>
        [C] =>
        [D] =>
        [E] =>
        [F] =>
        [G] =>
        [H] =>
        [I] =>
        [J] =>
    )
[3] => Array
    (
        [A] => 008850001000002098
        [B] => PG. DOC. 3 000113169  TP:NF FORNEC.:0000 38-PARANA EQUIPAMENTOS S.A.
        [C] =>
        [D] => Y1
        [E] =>
        [F] =>
        [G] =>
        [H] =>             2.188,35
        [I] =>
        [J] =>           8.901,01 C
    )

[4] => Array
    (
        [A] =>
        [B] =>
        [C] =>
        [D] =>
        [E] =>
        [F] =>
        [G] =>
        [H] =>
        [I] =>
        [J] =>
    )
</code>
)
Alguma ideia de como eu poderia fazer isto?
Caso eu tenha feito a pergunta de forma errada, essa é a primeira vez que pergunto.
Comment: Não entendi o que você quer fazer. Você quer pear só um dos elementos do array?
Comment: Esse array é montado pelo PHPExcel. A planilha tem mais registros, porém, coloquei somente estes como exemplo. Vou editar a pergunta com mais registro para tentar deixar mais claro o que estou tentando fazer.
Comment: não entendi o que você está tentando fazer também
Comment: O que não entendemos foi sua pergunta, "pegar um valor da linha seguine na linha anterior". Tente deixar sua dúvida mais clara pra podermos entender.
Answer: Não basta um laço de repetição?
Considerando que seu array esteja declarado dessa maneira :
<code>$arr = [
    [
        'A' => '000001001000003001',
        'B' => 'VLR REF COMPENSACAO DE VENDA CONFORME CO1',
        'C' => '1.1.2.01.020001',
        'D' => 'Z4',
        'E' => '',
        'F' => '',
        'G' => '',
        'H' => '',
        'I' => '100.000,00',
        'J' => '4.703.615,60 C'
    ],[
        'A' => '',
        'B' => 'MPRA DE EQUIPAMENTO DO FORNECEDOR PARANA 1',
        'C' => '',
        'D' => '',
        'E' => '',
        'F' => '',
        'G' => '',
        'H' => '',
        'I' => '',
        'J' => ''
    ],[
        'A' => '',
        'B' => 'EQUIPAMENTOS S.A. - NF VENDA 000000261 1',
        'C' => '',
        'D' => '',
        'E' => '',
        'F' => '',
        'G' => '',
        'H' => '',
        'I' => '',
        'J' => ''
    ]
];
</code>
Bastaria percorrer os arrays, concatenando os valores com o primeiro item do array, e então passando uma string vazia para o item atual.
<code>for ($i = 1; $i < count($arr); ++$i) {
    foreach ($arr[$i] as $chave => $valor) {
        if ($valor !== '') {
            $arr[0][$chave] .= " $valor";
            $arr[$i][$chave] = '';
        }
    }
}
</code>


Tyrus wss:// websocket not passing through squid proxy

Question: This is using Java 8, squid 3.1.2, and the tyrus-standalone-client-1.12.jarfor the websocket implementation.

UPDATE:
It is still not working with Tyrus.  However, it appears the problem is not with squid because when I make Chrome use the proxy and connect right here to SO, the connection to wss://qa.stackoverflow.com goes through the squid proxy and works fine.

I am using the test program given in the accepted answer to javax.websocket client simple example to try to get a squid proxy running.
However, I've changed it to use the websockets.org echo server and so the URI was changed from wss://real.okcoin.cn:10440/websocket/okcoinapi to wss://echo.websocket.org.
When I don't use a proxy the test program works fine with wss://echo.websocket.org and ws://echo.websocket.org.
When I specify a JVM-wide proxy via -Dhttp.proxyHost, -Dhttp.proxyPort, -Dhttps.proxyHost, and -Dhttps.proxyPort the ws:// URI works fine but the wss:// one does not.
I have verified (with tcpdump) that traffic is going to the proxy, that the test program is sending a CONNECT to the proxy, and that the proxy sends back a Connection established.  However, at that point nothing else happens:
# tcpdump -n -l -s 0 -S -X  'host 172.16.99.15'
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
17:21:45.361023 IP 172.16.99.21.36521 > 172.16.99.15.3128: S 1720321489:1720321489(0) win 14600 <mss 1460,sackOK,timestamp 1135431 0,nop,wscale 5>
        0x0000:  4500 003c 5696 4000 4006 c5e0 ac10 6315  E..<V.@.@.....c.
        0x0010:  ac10 630f 8ea9 0c38 668a 05d1 0000 0000  ..c....8f.......
        0x0020:  a002 3908 1e74 0000 0204 05b4 0402 080a  ..9..t..........
        0x0030:  0011 5347 0000 0000 0103 0305            ..SG........
17:21:45.361763 IP 172.16.99.15.3128 > 172.16.99.21.36521: S 1626710395:1626710395(0) ack 1720321490 win 14480 <mss 1460,sackOK,timestamp 1127109 1135431,nop,wscale 7>
        0x0000:  4500 003c 0000 4000 4006 1c77 ac10 630f  E..<..@.@..w..c.
        0x0010:  ac10 6315 0c38 8ea9 60f5 a17b 668a 05d2  ..c..8..`..{f...
        0x0020:  a012 3890 613d 0000 0204 05b4 0402 080a  ..8.a=..........
        0x0030:  0011 32c5 0011 5347 0103 0307            ..2...SG....
17:21:45.361789 IP 172.16.99.21.36521 > 172.16.99.15.3128: . ack 1626710396 win 457 <nop,nop,timestamp 1135431 1127109>
        0x0000:  4500 0034 5697 4000 4006 c5e7 ac10 6315  E..4V.@.@.....c.
        0x0010:  ac10 630f 8ea9 0c38 668a 05d2 60f5 a17c  ..c....8f...`..|
        0x0020:  8010 01c9 1e6c 0000 0101 080a 0011 5347  .....l........SG
        0x0030:  0011 32c5                                ..2.
17:21:45.407930 IP 172.16.99.21.36521 > 172.16.99.15.3128: P 1720321490:1720321613(123) ack 1626710396 win 457 <nop,nop,timestamp 1135442 1127109>
        0x0000:  4500 00af 5698 4000 4006 c56b ac10 6315  E...V.@.@..k..c.
        0x0010:  ac10 630f 8ea9 0c38 668a 05d2 60f5 a17c  ..c....8f...`..|
        0x0020:  8018 01c9 1ee7 0000 0101 080a 0011 5352  ..............SR
        0x0030:  0011 32c5 434f 4e4e 4543 5420 6563 686f  ..2.CONNECT.echo
        0x0040:  2e77 6562 736f 636b 6574 2e6f 7267 3a34  .websocket.org:4
        0x0050:  3433 2048 5454 502f 312e 310d 0a48 6f73  43.HTTP/1.1..Hos
        0x0060:  743a 2065 6368 6f2e 7765 6273 6f63 6b65  t:.echo.websocke
        0x0070:  742e 6f72 670d 0a50 726f 7879 2d43 6f6e  t.org..Proxy-Con
        0x0080:  6e65 6374 696f 6e3a 206b 6565 702d 616c  nection:.keep-al
        0x0090:  6976 650d 0a43 6f6e 6e65 6374 696f 6e3a  ive..Connection:
        0x00a0:  206b 6565 702d 616c 6976 650d 0a0d 0a    .keep-alive....
17:21:45.408347 IP 172.16.99.15.3128 > 172.16.99.21.36521: . ack 1720321613 win 114 <nop,nop,timestamp 1127121 1135442>
        0x0000:  4500 0034 93b9 4000 4006 88c5 ac10 630f  E..4..@.@.....c.
        0x0010:  ac10 6315 0c38 8ea9 60f5 a17c 668a 064d  ..c..8..`..|f..M
        0x0020:  8010 0072 c795 0000 0101 080a 0011 32d1  ...r..........2.
        0x0030:  0011 5352                                ..SR
17:21:45.423007 IP 172.16.99.15.3128 > 172.16.99.21.36521: P 1626710396:1626710435(39) ack 1720321613 win 114 <nop,nop,timestamp 1127124 1135442>
        0x0000:  4500 005b 93ba 4000 4006 889d ac10 630f  E..[..@.@.....c.
        0x0010:  ac10 6315 0c38 8ea9 60f5 a17c 668a 064d  ..c..8..`..|f..M
        0x0020:  8018 0072 bdab 0000 0101 080a 0011 32d4  ...r..........2.
        0x0030:  0011 5352 4854 5450 2f31 2e30 2032 3030  ..SRHTTP/1.0.200
        0x0040:  2043 6f6e 6e65 6374 696f 6e20 6573 7461  .Connection.esta
        0x0050:  626c 6973 6865 640d 0a0d 0a              blished....
17:21:45.423041 IP 172.16.99.21.36521 > 172.16.99.15.3128: . ack 1626710435 win 457 <nop,nop,timestamp 1135446 1127124>
        0x0000:  4500 0034 5699 4000 4006 c5e5 ac10 6315  E..4V.@.@.....c.
        0x0010:  ac10 630f 8ea9 0c38 668a 064d 60f5 a1a3  ..c....8f..M`...
        0x0020:  8010 01c9 1e6c 0000 0101 080a 0011 5356  .....l........SV
        0x0030:  0011 32d4                                ..2.
17:22:15.649132 IP 172.16.99.21.36521 > 172.16.99.15.3128: F 1720321613:1720321613(0) ack 1626710435 win 457 <nop,nop,timestamp 1143003 1127124>
        0x0000:  4500 0034 569a 4000 4006 c5e4 ac10 6315  E..4V.@.@.....c.
        0x0010:  ac10 630f 8ea9 0c38 668a 064d 60f5 a1a3  ..c....8f..M`...
        0x0020:  8011 01c9 1e6c 0000 0101 080a 0011 70db  .....l........p.
        0x0030:  0011 32d4                                ..2.
17:22:15.650241 IP 172.16.99.15.3128 > 172.16.99.21.36521: F 1626710435:1626710435(0) ack 1720321614 win 114 <nop,nop,timestamp 1134681 1143003>
        0x0000:  4500 0034 93bb 4000 4006 88c3 ac10 630f  E..4..@.@.....c.
        0x0010:  ac10 6315 0c38 8ea9 60f5 a1a3 668a 064e  ..c..8..`...f..N
        0x0020:  8011 0072 8c5b 0000 0101 080a 0011 5059  ...r.[........PY
        0x0030:  0011 70db                                ..p.
17:22:15.650255 IP 172.16.99.21.36521 > 172.16.99.15.3128: . ack 1626710436 win 457 <nop,nop,timestamp 1143003 1134681>
        0x0000:  4500 0034 569b 4000 4006 c5e3 ac10 6315  E..4V.@.@.....c.
        0x0010:  ac10 630f 8ea9 0c38 668a 064e 60f5 a1a4  ..c....8f..N`...
        0x0020:  8010 01c9 1e6c 0000 0101 080a 0011 70db  .....l........p.
        0x0030:  0011 5059                                ..PY

It's not that the proxy is having an issue with Java TLS connections.  If I instead (still using Java 8) make a vanilla HTTPS connection (say to https://www.google.com) through the proxy I see the same CONNECT being sent by the client and the same Connection established being sent back by the proxy, but then the TLS handshake, etc. proceed normally and the request/response finishes normally.
Frankly, I'm stumped as to what is going on (or rather not going on).
Answer: I was able to reproduce the issue 100% as described in the question. I used Squid version 3.5.17 (Windows 10 64bit),
a local Tomcat using a self-signed certificate with an "echo" web-socket and Java 7 (but I don't think Java 8 makes any difference).
Everything worked OK when I used org.glassfish.tyrus.bundles:tyrus-standalone-client:1.11 instead of 1.12.
I decided to use the previous version when I found this bugfix:
2015/Nov/13: Grizzly transport was unable to open wss connection via HTTP proxy
(github-commit).
Version 1.11 was released Jun 2015 and version 1.12 Sep 2015, I expect the fix to be in version 1.13
(see also the pom history).
The solution for this question might be related (2015/Aug/21).
For reference, the output from 1.12 (there is a hard-coded timeout of 30 seconds
in org.glassfish.tyrus.container.grizzly.client.GrizzlyClientContainer.CLIENT_SOCKET_TIMEOUT so you have to wait a while before the error appears).

05:07:25.514 [Grizzly(1) SelectorRunner] TRACE o.g.t.c.g.c.GrizzlyClientFilter - handleConnect
Using SSLEngineImpl.
Allow unsafe renegotiation: false
Allow legacy hello messages: true
Is initial handshake: true
Is secure renegotiation: false
05:07:25.651 [Grizzly(1)] DEBUG o.g.t.c.g.c.GrizzlyClientFilter - handleRead websocket: null content-size=0 headers=
HttpResponsePacket (
  status=200
  reason=Connection established
  protocol=HTTP/1.1
  content-length=-1
  committed=false
  headers=[]
)
javax.websocket.DeploymentException: Connection to 'wss://127.0.0.1:8443/wstest/ws/echoAsyncAnnotation' failed.
    at org.glassfish.tyrus.container.grizzly.client.GrizzlyClientSocket._connect(GrizzlyClientSocket.java:398)
    at org.glassfish.tyrus.container.grizzly.client.GrizzlyClientSocket.access$000(GrizzlyClientSocket.java:103)
    at org.glassfish.tyrus.container.grizzly.client.GrizzlyClientSocket$1.call(GrizzlyClientSocket.java:235)
    at org.glassfish.tyrus.container.grizzly.client.GrizzlyClientSocket$1.call(GrizzlyClientSocket.java:231)
    at org.glassfish.tyrus.container.grizzly.client.GrizzlyClientSocket.connect(GrizzlyClientSocket.java:249)
    at org.glassfish.tyrus.container.grizzly.client.GrizzlyClientContainer.openClientSocket(GrizzlyClientContainer.java:95)
    at org.glassfish.tyrus.client.ClientManager$3$1.run(ClientManager.java:663)
    at org.glassfish.tyrus.client.ClientManager$3.run(ClientManager.java:712)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
    at java.util.concurrent.FutureTask.run(FutureTask.java:262)
    at org.glassfish.tyrus.client.ClientManager$SameThreadExecutorService.execute(ClientManager.java:866)
    at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
    at org.glassfish.tyrus.client.ClientManager.connectToServer(ClientManager.java:511)
    at org.glassfish.tyrus.client.ClientManager.connectToServer(ClientManager.java:355)
    at nl.fw.wsclienttest.WsClient.echoDemo(WsClient.java:69)
    at nl.fw.wsclienttest.WsClient.main(WsClient.java:41)

And the code for the totally unsecure test-client I used:
import java.io.IOException;
import java.net.URI;
import java.security.KeyStore;
import java.security.cert.*;
import java.util.concurrent.*;

import javax.net.ssl.*;
import javax.websocket.*;

import org.glassfish.tyrus.client.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.bridge.SLF4JBridgeHandler;

public class WsClient {

    static {
        SLF4JBridgeHandler.removeHandlersForRootLogger();
        SLF4JBridgeHandler.install();
    }

    private static final Logger log = LoggerFactory.getLogger(WsClient.class);

    public static void main(String[] args) {

        System.getProperties().put("javax.net.debug", "ssl,handshake,data,sslctx");
        try {
            new WsClient().echoDemo();
            log.info("done");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    // localhost does not work via squid proxy
    String destUri = "wss://127.0.0.1:8443/wstest/ws/echoAsyncAnnotation";
    String proxyUri = "http://localhost:3128";
    volatile Session wsSession = null;

    void echoDemo() throws Exception {

        final ClientEndpointConfig cec = ClientEndpointConfig.Builder.create().build();
        ClientManager client = ClientManager.createClient();
        client.getProperties().put(ClientProperties.PROXY_URI, proxyUri);
        client.getProperties().put(ClientProperties.SHARED_CONTAINER, false);

        SSLContext sslCtx = createAllTrustingContext();
        SslEngineConfigurator sslEngineConfigurator = new SslEngineConfigurator(sslCtx, true, false, false);
        sslEngineConfigurator.setHostVerificationEnabled(false); //skip host verification
        client.getProperties().put(ClientProperties.SSL_ENGINE_CONFIGURATOR, sslEngineConfigurator);
        // time-out is set to 30 seconds for all operations, so handshake timeout does not work ...
        // see org.glassfish.tyrus.container.grizzly.client.GrizzlyClientContainer.CLIENT_SOCKET_TIMEOUT
        client.getProperties().put(ClientProperties.HANDSHAKE_TIMEOUT, 3000);
        final CountDownLatch messageLatch = new CountDownLatch(1);
        client.connectToServer(new Endpoint() {

            @Override
            public void onOpen(final Session session, EndpointConfig config) {
                try {
                    session.addMessageHandler(new MessageHandler.Whole<String>() {

                        @Override
                        public void onMessage(String message) {
                            log.info("Received message: {}", message);
                            wsSession = session;
                            messageLatch.countDown();
                        }
                    });
                    session.getBasicRemote().sendText("Hello world");
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }, cec, new URI(destUri));
        try {
            messageLatch.await(5, TimeUnit.SECONDS);
            if (wsSession != null) {
                wsSession.close();
            }
        } finally {
            client.shutdown();
        }
    }

    public static SSLContext createAllTrustingContext() throws Exception {

        SSLContext ctx = SSLContext.getInstance("TLSv1.2");
        KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
        kmf.init((KeyStore) null, "changeit".toCharArray());
        ctx.init(kmf.getKeyManagers(), new TrustManager[] { new TrustServerCertAlways() }, null);
        return ctx;
    }

    static class TrustServerCertAlways implements X509TrustManager {

        @Override public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
            log.debug("Trusting all client certificates.");
        }

        @Override public void checkServerTrusted(X509Certificate[] arg0, String arg1)   throws CertificateException {
            log.debug("Trusting all server certificates.");
        }

        @Override public X509Certificate[] getAcceptedIssuers() {
            log.debug("No accepted issuers.");
            return null;
        }
    }
}

Comment: Thanks!

1.11 worked fine.  I'll keep an eye out for 1.13.


Encoding Angle Data for Neural Network

Question: I am training a neural network (details not important) where the target data is a vector of angles (between 0 and 2*pi).  I am looking for advice on how to encode this data.  Here is what I am currently trying (with limited success):
1) 1-of-C encoding: I bin the set up possible angles into 1000 or so discrete angles and then indicate a particular angle by putting a 1 at the relevant index.  The problem with this, is that the network simply learns to output all 0's (since this is almost exactly correct).
2) Simple scaling: I scaled the networks output range ([0,1]) to [0,2*pi].  The problem here is that angles naturally have a circular topology (i.e. 0.0001 and 2*pi are actually right next to each other).  With this type of encoding, that information is lost.
Any suggestions would be appreciated!
Comment: A purely speculative encoding idea (I've not seen it done or tested it, but I haven't looked) is the encode your angle ($\theta$) as a pair: $\theta \mapsto (\sin(\theta),\cos(\theta))$. I think then it would be a continous map with all values like $0$, and $2\pi$ close to each other.
I think I might build a demo of this and test it out.
Comment: You should not be having a problem with the network outputting all zeros, if you use a softmax output layer -- which you generally should do, if you are using a catagorical (i.e. 1-of-C) output.
Comment: i've been thinking about this some more, and I think it might actually jsut be all in your loss function. I want to try a bunch of things. I made the demo, but didn't finish tests on it. Expect a detailed answer with experimental backing tomorrow sometime. (Poke me if I don't)
Comment: I am not currently using a softmax layer, and that is probably the problem.  I'll implement that today if I get a chance!  Your (cos,sin) idea is very interesting and I especially like that it automatically puts that range into [-1,1] (good if you are working with a tanh activation function).  I look forward to seeing your results1
Comment: A quick update: I tried implementing a softmax layer, and am still having no luck.  The issue, I think, is that for this problem it is essential that the "angularity" of the data be represented somehow in the encoding.  With a categorical encoding, the topology of the target data is lost.  Hence an error of 0.5*pi and 0.05*pi look the same to the network (it sees both as incorrect categorizations).
Answer: Introduction
I find this question really interesting, I'm assume someone has put out a paper on it, but it's my day off, so I don't want to go chasing references.
So we could consider it as an representation/encoding of the output, which I do in this answer.
I remain thinking that there is a better way, where you can just use a slightly different loss function.
(Perhaps sum of squared differences, using subtraction modulo 2 $\pi$).
But onwards with the actual answer.
Method
I propose that an angle $\theta$ be represented as a pair of values, its sine and its cosine.
So the encoding function is:   $\qquad\qquad\quad\theta \mapsto (\sin(\theta), \cos(\theta))$
and the decoding function is: $\qquad(y_1,y_2) \mapsto \arctan\!2(y_1,y_2)$
For arctan2 being the inverse tangents, preserving direction in all quadrants)

You could, in theory, equivalently work directly with the angles if your tool use supported atan2 as a layer function (taking exactly 2 inputs and producing 1 output).
TensorFlow does this now, and supports gradient descent on it, though not it intended for this use.
I investigated using out = atan2(sigmoid(ylogit), sigmoid(xlogit))  with a loss function min((pred - out)^2, (pred - out - 2pi)^2).
I found that it trained far worse than
using outs = tanh(ylogit), outc = tanh(xlogit))  with a loss function 0.5((sin(pred) - outs)^2 + (cos(pred) - outc)^2.
Which I think can be attributed to the gradient being discontinuous for atan2

My testing here runs it as a preprocessing function
To evaluate this I defined a task:

Given a black and white image representing a single line on a blank background
  Output what angle that line is at to the "positive x-axis"

I implemented a function randomly generate these images, with lines at random angles (NB: earlier versions of this post used random slopes, rather than random angles. Thanks to @Ari Herman for point it out. It is now fixed).
I constructed several neural networks to evaluate there performance on the task. The full details of implementation are in this Jupyter notebook.
The code is all in Julia, and I make use of the Mocha neural network library.
For comparison, I present it against the alternative methods of scaling to 0,1.
and to putting into 500 bins and using soft-label softmax.
I am not particularly happy with the last, and feel I need to tweak it.
Which is why, unlike the others I only trial it for 1,000 iterations, vs the other two which were run for 1,000 and for 10,000
Experimental Setup
Images were $101\times101$ pixels, with the line commensing at the center and going to the edge.
There was no noise etc in the image, just a "black" line, on a white background.
For each trail 1,000 training, and 1,000 test images were generated randomly.
The evaluation network had a single hidden layer of width 500.
Sigmoid neurons were used in the hidden layer.
It was trained by Stochastic Gradient Decent, with a fixed learning rate of 0.01, and a fixed momentum of 0.9.
No regularization, or dropout was used. Nor was any kind of convolution etc.
A simple network, which I hope suggests that these results will generalize
It is very easy to tweak these parameters in the test code, and I encourage people to do so. (and look for bugs in the test).
Results
My results are as follows:
|                        |  500 bins    |  scaled to 0-1 |  Sin/Cos     |  scaled to 0-1 |  Sin/Cos     |
|                        | 1,000 Iter   | 1,000 Iter     | 1,000 iter   | 10,000 Iter    | 10,000 iter  |
|------------------------|--------------|----------------|--------------|----------------|--------------|
| mean_error             | 0.4711263342 | 0.2225284486   | 2.099914718  | 0.1085846429   | 2.1036656318 |
| std(errors)            | 1.1881991421 | 0.4878383767   | 1.485967909  | 0.2807570442   | 1.4891605068 |
| minimum(errors)        | 1.83E-006    | 1.82E-005      | 9.66E-007    | 1.92E-006      | 5.82E-006    |
| median(errors)         | 0.0512168533 | 0.1291033982   | 1.8440767072 | 0.0562908143   | 1.8491085947 |
| maximum(errors)        | 6.0749693965 | 4.9283551248   | 6.2593307366 | 3.735884823    | 6.2704853962 |
| accurancy              | 0.00%        | 0.00%          | 0.00%        | 0.00%          | 0.00%        |
| accurancy_to_point001  | 2.10%        | 0.30%          | 3.70%        | 0.80%          | 12.80%       |
| accurancy_to_point01   | 21.90%       | 4.20%          | 37.10%       | 8.20%          | 74.60%       |
| accurancy_to_point1    | 59.60%       | 35.90%         | 98.90%       | 72.50%         | 99.90%       |

Where I refer to error, this is the absolute value of the difference between the angle output by the neural network, and the true angle. So the mean error (for example) is the average over the 1,000 test cases of this difference etc.
I am not sure that I should not be rescaling it by making an error of say $\frac{7\pi}{4}$ be equal to an error of $\frac{\pi}{4}$).
I also present the accuracy at various levels of granularity.
The accuracy being the portion of test cases it got corred.
So accuracy_to_point01 means that it was counted as correct if the output was within 0.01 of the true angle.
None of the representations got any perfect results, but that is not at all surprising given how floating point math works.
If you take a look at the history of this post you will see the results do have a bit of noise to them, slightly different each time I rerun it. But the general order and scale of values remains the same; thus allowing us to draw some conclusions.
Discussion
Binning with softmax performs by far the worst, as I said I am not sure I didn't screw up something in the implementation.
It does perform marginally above the guess rate though. if it were just guessing we would be getting a mean error of $\pi$
The sin/cos encoding performs significantly better than the scaled 0-1 encoding.
The improvement is to the extent that at 1,000 training iterations sin/cos is performing about 3 times better on most metrics than scaling is at 10,000 iterations.
I think, in part this is related to improving generalization,
as both were getting fairly similar mean squared error on the training set, at least once 10,000 iterations were run.
There is certainly an upper limit on the best possible performance at this task, given that the Angle could be more or less any real number, but not all such angels produce different lines at the resolution of $101\times101$ pixels.
So since, for example, the angles 45.0 and 45.0000001 both are tied to the same image at that resolution, no method will ever get both perfectly correct.
It also seems likely that on an absolute scale to move beyond this performance, a better neural network is needed. Rather than the very simple one outlined above in experimental setup.
Conclusion.
It seems that the sin/cos representation is by far the best, of the representations I investigated here. This does make sense, in that it does have a smooth value as you move around the circle.
I also like that the inverse can be done with arctan2, which is elegant.
I believe the task presented is sufficient in its ability to present a reasonable challenge for the network. Though I guess really it is just learning to do curve fitting to $f(x)=\frac{y1}{y2} x$ so perhaps it is too easy.
And perhaps worse it may be favouring the paired representation.
I don't think it is, but it is getting late here, so I might have missed something
I invite you again to look over my code.
Suggest improvements, or alternative tasks.
Comment: This is certainly the most thorough response I have ever received on stack exchange.  Since I am not familiar with Julia, it is difficult for me to examine your code...so, instead I am going to try to replicate your results using Python.  I'll post me findings later today or tomorrow.
Comment: While I was not surprised that binning performed poorly, I was surprised at the degree to which (0,1) scaling was outperformed by the (cos,sin) method.  I noticed that you generated your examples by randomly picking the rise and run of the lines.  This would, I think, generate lines whose angles are not uniformly distributed, but whose slopes are.  Is it possible that this is why the (cos,sin) method performed so much better?    What would happen if you made the targets tan(angle)...?
Comment: you're right about random slope vs random angle. but I don't think targetting `tan(angle)` will go so well, given than tan isn't defined for all angles (eg $\pi/4$). I will rerun it with randomly generated angles and edit the posts.
Comment: There should be a close to one to one map between julia and numpy, and between Mocha and Caffe, if you really want to reimplement it. Is there a particular part of the code you find hard to read? Julia should be an easy to understand language. So perhaps I have done something odd.
Comment: I did end up reading your code, and everything seems correct.  Still, I wanted to write my own version, since doing so is usually instructive.  My implementation is slightly different from yours, so it will be interesting to compare results.  I will be posting them within the next couple of hours.
Comment: You are right, rewriting (and refactoring) to understand are great ways of comprehending how something actually works. I started to tweak the code to do random angles rather than random slopes, but it is WIP, and i got to do some work. I am interested to see your results too.
Comment: @AriHerman Code and Table have now been updated. You can see that changing random slope for random angle has improved performance across the board, particularly for the scaled approach, but not by enough to  change any of the conclusions.
Comment: Thank you Ari and Lyndon for the helpful comments. @LyndonWhite , you mentioned that you think using a different loss function would be a better approach - does using `arctan2` make sense as a loss function directly?
Comment: One can't use arc tan directly as a loss function (it doesn't capture any sense of loss) and as per my edits yesterday I found it worked badly as an activation function (or output function depending on how defined)
Comment: How does the logit come along? how do I make sure logit.
Comment: I don't think the sigmoid in the `out = atan2(sigmoid(ylogit), sigmoid(xlogit))` experiment is reasonable. It only allows angles in the upper right quadrant, while heavily prioritizing those closer to pi/4. Bad peformance is not surprising.
Comment: This is an amazing answer and exactly what I was looking for. Thank you!  Small +1 to Aydo here. Sigmoid only outputs in [0,1] - that explains the bad performance. We need [-1, 1]. It appears that OP made the same error initially and got much better results after fixing that problem. But in fact, we don't even need the tanh or sigmoid, as atan2 is defined for all inputs and only uses their ratio. @airalcorn2 directly uses MSE loss on the linear layer output with no activation function.
Answer: Here's another Python implementation comparing Lyndon White's proposed encoding to a binned approach. The code below produced the following output:
Training Size: 100
Training Epochs: 100
Encoding: cos_sin
Test Error: 0.017772154610047136
Encoding: binned
Test Error: 0.043398792553251526

Training Size: 100
Training Epochs: 500
Encoding: cos_sin
Test Error: 0.015376604917819397
Encoding: binned
Test Error: 0.032942592915322394

Training Size: 1000
Training Epochs: 100
Encoding: cos_sin
Test Error: 0.007544091937411164
Encoding: binned
Test Error: 0.012796594492198667

Training Size: 1000
Training Epochs: 500
Encoding: cos_sin
Test Error: 0.0038051515079569097
Encoding: binned
Test Error: 0.006180633805557207

As you can see, while the binned approach performs admirably in this toy task, the $(\sin(\theta), \cos(\theta))$ encoding performs better in all training configurations, sometimes by a considerable margin. I suspect as the specific task became more complex, the benefits of using Lyndon White's $(\sin(\theta), \cos(\theta))$ representation would become more pronounced.
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torch.utils.data

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class Net(nn.Module):
    def __init__(self, input_size, hidden_size, num_out):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.sigmoid = nn.Sigmoid()
        self.fc2 = nn.Linear(hidden_size, num_out)

    def forward(self, x):
        out = self.fc1(x)
        out = self.sigmoid(out)
        out = self.fc2(out)
        return out

def gen_train_image(angle, side, thickness):
    image = np.zeros((side, side))
    (x_0, y_0) = (side / 2, side / 2)
    (c, s) = (np.cos(angle), np.sin(angle))
    for y in range(side):
        for x in range(side):
            if (abs((x - x_0) * c + (y - y_0) * s) < thickness / 2) and (
                    -(x - x_0) * s + (y - y_0) * c > 0):
                image[x, y] = 1

    return image.flatten()

def gen_data(num_samples, side, num_bins, thickness):
    angles = 2 * np.pi * np.random.uniform(size=num_samples)
    X = [gen_train_image(angle, side, thickness) for angle in angles]
    X = np.stack(X)

    y = {"cos_sin": [], "binned": []}
    bin_size = 2 * np.pi / num_bins
    for angle in angles:
        idx = int(angle / bin_size)
        y["binned"].append(idx)
        y["cos_sin"].append(np.array([np.cos(angle), np.sin(angle)]))

    for enc in y:
        y[enc] = np.stack(y[enc])

    return (X, y, angles)

def get_model_stuff(train_y, input_size, hidden_size, output_sizes,
                    learning_rate, momentum):
    nets = {}
    optimizers = {}

    for enc in train_y:
        net = Net(input_size, hidden_size, output_sizes[enc])
        nets[enc] = net.to(device)
        optimizers[enc] = torch.optim.SGD(net.parameters(), lr=learning_rate,
                                          momentum=momentum)

    criterions = {"binned": nn.CrossEntropyLoss(), "cos_sin": nn.MSELoss()}
    return (nets, optimizers, criterions)

def get_train_loaders(train_X, train_y, batch_size):
    train_X_tensor = torch.Tensor(train_X)

    train_loaders = {}

    for enc in train_y:
        if enc == "binned":
            train_y_tensor = torch.tensor(train_y[enc], dtype=torch.long)
        else:
            train_y_tensor = torch.tensor(train_y[enc], dtype=torch.float)

        dataset = torch.utils.data.TensorDataset(train_X_tensor, train_y_tensor)
        train_loader = torch.utils.data.DataLoader(dataset=dataset,
                                                   batch_size=batch_size,
                                                   shuffle=True)
        train_loaders[enc] = train_loader

    return train_loaders

def show_image(image, side):
    img = plt.imshow(np.reshape(image, (side, side)), interpolation="nearest",
                     cmap="Greys")
    plt.show()

def main():
    side = 101
    input_size = side ** 2
    thickness = 5.0
    hidden_size = 500
    learning_rate = 0.01
    momentum = 0.9
    num_bins = 500
    bin_size = 2 * np.pi / num_bins
    half_bin_size = bin_size / 2
    batch_size = 50
    output_sizes = {"binned": num_bins, "cos_sin": 2}
    num_test = 1000

    (test_X, test_y, test_angles) = gen_data(num_test, side, num_bins,
                                             thickness)

    for num_train in [100, 1000]:

        (train_X, train_y, train_angles) = gen_data(num_train, side, num_bins,
                                                    thickness)
        train_loaders = get_train_loaders(train_X, train_y, batch_size)

        for epochs in [100, 500]:

            (nets, optimizers, criterions) = get_model_stuff(train_y, input_size,
                                                             hidden_size, output_sizes,
                                                             learning_rate, momentum)

            for enc in train_y:
                optimizer = optimizers[enc]
                net = nets[enc]
                criterion = criterions[enc]

                for epoch in range(epochs):
                    for (i, (images, ys)) in enumerate(train_loaders[enc]):
                        optimizer.zero_grad()

                        outputs = net(images.to(device))
                        loss = criterion(outputs, ys.to(device))
                        loss.backward()
                        optimizer.step()

            print("Training Size: {0}".format(num_train))
            print("Training Epochs: {0}".format(epochs))
            for enc in train_y:
                net = nets[enc]
                preds = net(torch.tensor(test_X, dtype=torch.float).to(device))
                if enc == "binned":
                    pred_bins = np.array(preds.argmax(dim=1).detach().cpu().numpy(),
                                         dtype=np.float)
                    pred_angles = bin_size * pred_bins + half_bin_size
                else:
                    pred_angles = torch.atan2(preds[:, 1], preds[:, 0]).detach().cpu().numpy()
                    pred_angles[pred_angles < 0] = pred_angles[pred_angles < 0] + 2 * np.pi

                print("Encoding: {0}".format(enc))
                print("Test Error: {0}".format(np.abs(pred_angles - test_angles).mean()))

            print()

if __name__ == "__main__":
    main()

Answer: Here is my Python version of your experiment.  I kept many of the details of your implementation the same, in particular I use the same image size, network layer sizes, learning rate, momentum, and success metrics.
Each network tested has one hidden layer (size = 500) with logistic neurons.  The output neurons are either linear or softmax as noted.  I used 1,000 training images and 1,000 test images which were independently, randomly generated (so there may be repeats).  Training consisted of 50 iterations through the training set.
I was able to get quite good accuracy using binning and "gaussian" encoding (a name I made up; similar to binning except that the target output vector has the form exp(-pi*([1,2,3,...,500] - idx)**2) where idx is the index corresponding to the correct angle).  The code is below; here are my results:
Test error for (cos,sin) encoding:
1,000 training images, 1,000 test images, 50 iterations, linear output

Mean: 0.0911558142071
Median: 0.0429723541743
Minimum: 2.77769843793e-06
Maximum: 6.2608513539
Accuracy to 0.1: 85.2%
Accuracy to 0.01: 11.6%
Accuracy to 0.001: 1.0%

Test error for [-1,1] encoding:
1,000 training images, 1,000 test images, 50 iterations, linear output

Mean: 0.234181700523
Median: 0.17460197307
Minimum: 0.000473665840258
Maximum: 6.00637777237
Accuracy to 0.1: 29.9%
Accuracy to 0.01: 3.3%
Accuracy to 0.001: 0.1%

Test error for 1-of-500 encoding:
1,000 training images, 1,000 test images, 50 iterations, softmax output

Mean: 0.0298767021922
Median: 0.00388858079174
Minimum: 4.08712407829e-06
Maximum: 6.2784479965
Accuracy to 0.1: 99.6%
Accuracy to 0.01: 88.9%
Accuracy to 0.001: 13.5%

Test error for gaussian encoding:
1,000 training images, 1,000 test images, 50 iterations, softmax output

Mean: 0.0296905377463
Median: 0.00365867335107
Minimum: 4.08712407829e-06
Maximum: 6.2784479965
Accuracy to 0.1: 99.6%
Accuracy to 0.01: 90.8%
Accuracy to 0.001: 14.3%

I cannot figure out why our results seem to be in contradiction with one another, but it seems worth further investigation.
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 13 16:59:53 2016

@author: Ari
"""

from numpy import savetxt, loadtxt, round, zeros, sin, cos, arctan2, clip, pi, tanh, exp, arange, dot, outer, array, shape, zeros_like, reshape, mean, median, max, min
from numpy.random import rand, shuffle
import matplotlib.pyplot as plt

###########
# Functions
###########

# Returns a B&W image of a line represented as a binary vector of length width*height
def gen_train_image(angle, width, height, thickness):
    image = zeros((height,width))
    x_0,y_0 = width/2, height/2
    c,s = cos(angle),sin(angle)
    for y in range(height):
        for x in range(width):
            if abs((x-x_0)*c + (y-y_0)*s) < thickness/2 and -(x-x_0)*s + (y-y_0)*c > 0:
                image[x,y] = 1
    return image.flatten()

# Display training image
def display_image(image,height, width):
    img = plt.imshow(reshape(image,(height,width)), interpolation = 'nearest', cmap = "Greys")
    plt.show()

# Activation function
def sigmoid(X):
    return 1.0/(1+exp(-clip(X,-50,100)))

# Returns encoded angle using specified method ("binned","scaled","cossin","gaussian")
def encode_angle(angle, method):
    if method == "binned": # 1-of-500 encoding
        X = zeros(500)
        X[int(round(250*(angle/pi + 1)))%500] = 1
    elif method == "gaussian": # Leaky binned encoding
        X = array([i for i in range(500)])
        idx = 250*(angle/pi + 1)
        X = exp(-pi*(X-idx)**2)
    elif method == "scaled": # Scaled to [-1,1] encoding
        X = array([angle/pi])
    elif method == "cossin": # Oxinabox's (cos,sin) encoding
        X = array([cos(angle),sin(angle)])
    else:
        pass
    return X

# Returns decoded angle using specified method
def decode_angle(X, method):
    if method == "binned" or method == "gaussian": # 1-of-500 or gaussian encoding
        M = max(X)
        for i in range(len(X)):
            if abs(X[i]-M) < 1e-5:
                angle = pi*i/250 - pi
                break
#        angle = pi*dot(array([i for i in range(500)]),X)/500  # Averaging
    elif method == "scaled": # Scaled to [-1,1] encoding
        angle = pi*X[0]
    elif method == "cossin": # Oxinabox's (cos,sin) encoding
        angle = arctan2(X[1],X[0])
    else:
        pass
    return angle

# Train and test neural network with specified angle encoding method
def test_encoding_method(train_images,train_angles,test_images, test_angles, method, num_iters, alpha = 0.01, alpha_bias = 0.0001, momentum = 0.9, hid_layer_size = 500):
    num_train,in_layer_size = shape(train_images)
    num_test = len(test_angles)

    if method == "binned":
        out_layer_size = 500
    elif method == "gaussian":
        out_layer_size = 500
    elif method == "scaled":
        out_layer_size = 1
    elif method == "cossin":
        out_layer_size = 2
    else:
        pass

    # Initial weights and biases
    IN_HID = rand(in_layer_size,hid_layer_size) - 0.5 # IN --> HID weights
    HID_OUT = rand(hid_layer_size,out_layer_size) - 0.5 # HID --> OUT weights
    BIAS1 = rand(hid_layer_size) - 0.5 # Bias for hidden layer
    BIAS2 = rand(out_layer_size) - 0.5 # Bias for output layer

    # Initial weight and bias updates
    IN_HID_del = zeros_like(IN_HID)
    HID_OUT_del = zeros_like(HID_OUT)
    BIAS1_del = zeros_like(BIAS1)
    BIAS2_del = zeros_like(BIAS2)

    # Train
    for j in range(num_iters):
        for i in range(num_train):
            # Get training example
            IN = train_images[i]
            TARGET = encode_angle(train_angles[i],method)

            # Feed forward and compute error derivatives
            HID = sigmoid(dot(IN,IN_HID)+BIAS1)

            if method == "binned" or method == "gaussian": # Use softmax
                OUT = exp(clip(dot(HID,HID_OUT)+BIAS2,-100,100))
                OUT = OUT/sum(OUT)
                dACT2 = OUT - TARGET
            elif method == "cossin" or method == "scaled": # Linear
                OUT = dot(HID,HID_OUT)+BIAS2
                dACT2 = OUT-TARGET
            else:
                print("Invalid encoding method")

            dHID_OUT = outer(HID,dACT2)
            dACT1 = dot(dACT2,HID_OUT.T)*HID*(1-HID)
            dIN_HID = outer(IN,dACT1)
            dBIAS1 = dACT1
            dBIAS2 = dACT2

            # Update the weight updates
            IN_HID_del = momentum*IN_HID_del + (1-momentum)*dIN_HID
            HID_OUT_del = momentum*HID_OUT_del + (1-momentum)*dHID_OUT
            BIAS1_del = momentum*BIAS1_del + (1-momentum)*dBIAS1
            BIAS2_del = momentum*BIAS2_del + (1-momentum)*dBIAS2

            # Update the weights
            HID_OUT -= alpha*dHID_OUT
            IN_HID -= alpha*dIN_HID
            BIAS1 -= alpha_bias*dBIAS1
            BIAS2 -= alpha_bias*dBIAS2

    # Test
    test_errors = zeros(num_test)
    angles = zeros(num_test)
    target_angles = zeros(num_test)
    accuracy_to_point001 = 0
    accuracy_to_point01 = 0
    accuracy_to_point1 = 0

    for i in range(num_test):

        # Get training example
        IN = test_images[i]
        target_angle = test_angles[i]

        # Feed forward
        HID = sigmoid(dot(IN,IN_HID)+BIAS1)

        if method == "binned" or method == "gaussian":
            OUT = exp(clip(dot(HID,HID_OUT)+BIAS2,-100,100))
            OUT = OUT/sum(OUT)
        elif method == "cossin" or method == "scaled":
            OUT = dot(HID,HID_OUT)+BIAS2

        # Decode output
        angle = decode_angle(OUT,method)

        # Compute errors
        error = abs(angle-target_angle)
        test_errors[i] = error
        angles[i] = angle

        target_angles[i] = target_angle
        if error < 0.1:
            accuracy_to_point1 += 1
        if error < 0.01:
            accuracy_to_point01 += 1
        if error < 0.001:
            accuracy_to_point001 += 1

    # Compute and return results
    accuracy_to_point1 = 100.0*accuracy_to_point1/num_test
    accuracy_to_point01 = 100.0*accuracy_to_point01/num_test
    accuracy_to_point001 = 100.0*accuracy_to_point001/num_test

    return mean(test_errors),median(test_errors),min(test_errors),max(test_errors),accuracy_to_point1,accuracy_to_point01,accuracy_to_point001

# Dispaly results
def display_results(results,method):
    MEAN,MEDIAN,MIN,MAX,ACC1,ACC01,ACC001 = results
    if method == "binned":
        print("Test error for 1-of-500 encoding:")
    elif method == "gaussian":
        print("Test error for gaussian encoding: ")
    elif method == "scaled":
        print("Test error for [-1,1] encoding:")
    elif method == "cossin":
        print("Test error for (cos,sin) encoding:")
    else:
        pass
    print("-----------")
    print("Mean: "+str(MEAN))
    print("Median: "+str(MEDIAN))
    print("Minimum: "+str(MIN))
    print("Maximum: "+str(MAX))
    print("Accuracy to 0.1: "+str(ACC1)+"%")
    print("Accuracy to 0.01: "+str(ACC01)+"%")
    print("Accuracy to 0.001: "+str(ACC001)+"%")
    print("\n\n")

##################
# Image parameters
##################
width = 100 # Image width
height = 100 # Image heigth
thickness = 5.0 # Line thickness

#################################
# Generate training and test data
#################################
num_train = 1000
num_test = 1000
test_images = []
test_angles = []
train_images = []
train_angles = []
for i in range(num_train):
    angle = pi*(2*rand() - 1)
    train_angles.append(angle)
    image = gen_train_image(angle,width,height,thickness)
    train_images.append(image)
for i in range(num_test):
    angle = pi*(2*rand() - 1)
    test_angles.append(angle)
    image = gen_train_image(angle,width,height,thickness)
    test_images.append(image)
train_angles,train_images,test_angles,test_images = array(train_angles),array(train_images),array(test_angles),array(test_images)

###########################
# Evaluate encoding schemes
###########################
num_iters = 50

# Train with cos,sin encoding
method = "cossin"
results1 = test_encoding_method(train_images, train_angles, test_images, test_angles, method, num_iters)
display_results(results1,method)

# Train with scaled encoding
method = "scaled"
results3 = test_encoding_method(train_images, train_angles, test_images, test_angles, method, num_iters)
display_results(results3,method)

# Train with binned encoding
method = "binned"
results2 = test_encoding_method(train_images, train_angles, test_images, test_angles, method, num_iters)
display_results(results2,method)

# Train with gaussian encoding
method = "gaussian"
results4 = test_encoding_method(train_images, train_angles, test_images, test_angles, method, num_iters)
display_results(results4,method)

Comment: Cool, On key different. You are only training on each image once. I am training on each image 1,000 times, or 10,000 times. Multiple Iterations though the training data are normal, particularly when training on relatively small amounts of data (and it only took me one unpublishable undergrad thesis to learn this, but that is another story).
With that said, I should add a 1 iter column to my table. that would be informative
Comment: I would think that training on similar (but not identical) images with similar targets would affect that network similarly.  If that is true, it should work fine to simply increase the number of random images being trained on, rather iterating many times through a smaller training set.  Are you saying this is not the case?
Comment: It is similar, but for this example task it doesn't has the issue that eventually you will show all possible images so your test will overlap with your train, so testing generalistion won't work.
More significantly though you do 100,000 training images, which is <1000*1000 training images*Iterations.
Comment: You are correct, I will fix that issue.  There is an even more significant problem with my code: I am using logistic neurons which are incapable of producing the negative values required by the (cos,sin) representation.  D'oh!  I will be revising my code and re-posting as soon as possible.
Comment: You might (if you haven't already done so) be interested in doing a [Graident Check](https://youtu.be/KgZ4ZTTfKO8), which is worthwhile when implemented neural networks from scratch, as it is very easy to make a minor mistake and have your network still mostly work.

Re: Neuron: yeah, I have a linear output layer, onto of the sigmoid hidden layer
Comment: @Oxinabox Sorry for taking so long to re-post.  My code is less efficient than yours, and is not done running.  However, my preliminary results appear to contrast with yours.  I was able to get 99% accuracy (to 1 decimal) using categorical encoding and only 50,000 images*iterations.  The program should finish later today, so hopefully I will be able to post tonight.  Also, if you are interested in corresponding further about the question of encoding angular data, feel free to e-mail me: email@example.com; I will be continuing to work on this throughout the summer.
Comment: I have posted the new results.  This time I used linear output neurons for the networks that use "scaled" and "sin, cos" encoding.  I am also now using images with lines that go out in one direction from the center, so it is more similar to yours.
Comment: I think I know what is wrong with my Binned outputs, I am using Find to find the maximum rather than FindMax, find does not find the maximum, but rather the first nonzero so it will be wrong since it will always find the first entry to be nonzero under softmax.
Your method on the other hand also doesn't find the max, instead finds the weighed average angle, which is either a great idea, or a not-so-great idea. Great if there is only one angle it peaks at -- that lets it get potentially better accurasy than the bins provide, not so great if there are 2+ peaks.
Comment: In general our results agree except for for binned, and as I said in my post, I was not too sure on the correctness of my binned approach, and see last comment I now know my binned approach was wrong.
I'll update mine and repost, at some-point.
Comment: I toyed with the idea of using the weighted average (and I have a line that does that, but it is commented out).  I believe my code IS finding the maximum...could you point me to the specific spot you are referring to?
Comment: Oh so you did, I didn't see the weighted average line was commented out.
Comment: @AriHerman - going through both codes and looking for discrepancies  - not sure you noticed this - but you're using 100 x 100 dimensions not 101 x 101. I'm attempting to port to swift - https://github.com/johndpope/Bresenham-Line with neural net.
Answer: Another way to encode the angle is as a set of two values:

y1 = max(0,theta)
y2 = max(0,-theta)
theta_out = y1 - y2

This would have the similar problem to arctan2 in that the gradient is undefined at theta = 0. I don't have the time to train a network and compare to the other encodings but in this paper the technique seemed reasonably successful.
Comment: This seems like an answer mixed in with another question in one post. This site operates a bit differently from the way forum would. Here answers should concentrate on answering the original question. And if you have another question or a comment - it should be posted as such.

Encoding Angle Data for Neural Network


How to prepare directories in private file system?

Question: I've configured private file system. Using it from UI works well.
I want to generate dump site content via devel_generate with custom modifications. So I'm hooking into Node API for assigning sample PDF files to nodes.
Inside private directory I want to create directory structure:
<code>  $dir = sprintf('private://org%d/%d/person%d', $this->getOrganization()->nid,
                    date('Y'), $this->getEmployee()->uid);

  if (!file_prepare_directory($dir)) {
    throw new Exception('Cannot create document file directory or it\'s not writable');
  }
</code>
But exception is thrown.
How to set proper directory permissions? Content is generated  via drush command - drush genc.
Comment: You need to run Drush under the same user your web server runs, or give Drush write permissions over the private file system
Comment: Thanks, I'm trying things like: sudo -uwww-data /usr/bin/drush cc all
but getting drush command not found
Comment: I've same error when creating nodes via UI also...
Comment: Sorry, would need access to your server to be able to debug, can't do it from here
Answer: Ok, I solved it by passing proper flag (for UI):
<code>file_prepare_directory($dir, FILE_CREATE_DIRECTORY);
</code>
For using drush command still don't know and failing.


Progressive huffman table

Question: Do anyone of you know of a defacto standard huffman table for progressive jpeg encoding? There's examples of tables for baseline encoding in the standard that a lot of implementations use that seem to be adequate for most uses in regards of symbol frequency and so on but I can't seem to find one for progressive encoding that includes the symbols EOB1 through EOB14.
Answer: I am not aware of such a table. The approach is to do two passes over the data and find the best table for the scan in questions.

Progressive huffman table