The whole process mainly depends on the ProcessFile() function:I get all the credentials and details from a JSON file.Initially I get data from the db: two IDs and a URL.There is a text file for address state matching, reading, and compiling with | to match regex.The same is done for zip and zip keywords.Iterate over the row.Get content of the URL and remove unwanted things.Match the address regex and store the result to a variable.Match the zip regex, see if it is available in the zip keyword and write the result to variable append the total results to outputs list. Write only when the list size is equal or greater than 1000.After completing all the URLs, see if outputs contains data. If so, write to the file, else leave it.Move the result to another FTP location.#Importing modulesimport requestsimport reimport sysimport pyodbcimport jsonimport datetimeimport randomimport timeimport ftplibimport traceback#loading regex filesstart_time_split=datetime.datetime.now()json_data=open('server_details.json')server_details = json.load(json_data)json_data.close()   '''SQL SERVER CREDENTIALS AND DB NAME'''SERVER=server_details[SERVER]DATABASE=server_details[DATABASE]UID=server_details[UID]PWD=server_details[PWD]ADDRESS=server_details[ADDRESS]ZIP=server_details[ZIP]ZIP_WORD=server_details[ZIP_WORD]FTP_IP=server_details[FTP_IP]FTP_USER=server_details[FTP_USER]FTP_PWD=server_details[FTP_PWD]FTP_LOCATION=server_details[FTP_LOCATION]def ProcessFile(fileReadStr):    try:        header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0'}        with  open(ADDRESS,r) as l:            address=|.join([a.strip() for a in l])        #address=address                with  open(ZIP,r) as ll:            regex=|.join([a.strip() for a in ll])        #regex=regex        with open(ZIP_WORD) as inp:            zip_word=[a.strip() for a in inp.readlines()]        print Started +str(start_time_split)        filename=str(random.randint(1,100000))+_+str(int(time.time()))+_us_link_classifier.txt        with open(filename,w) as  output:            output.write(ID\tClientID\tURL\tAddress\tword\tzip\tzipmatch\n)        outputs=[]        for line in fileReadStr:            id1=line[0]            web=line[1] if line[1].strip().startswith(http) else http://+line[1].strip()            id2=line[2]              ad=0            word=None              try:                objReq = requests.get(web,timeout=30,verify=False,allow_redirects=True,headers=header)                htmlStr = objReq.content                htmlStr=re.sub(\s+, ,htmlStr)                htmlStr=re.sub(<!--.*?-->,  , htmlStr)                htmlStr=re.sub(<head.*?>.*?</head>, ,htmlStr)                htmlStr=re.sub(<.*?>,  , htmlStr)                   htmlStr=re.sub(\s+,  , htmlStr)                address_match=re.search(address,htmlStr,re.IGNORECASE)                if address_match :                    ad=1                    word=address_match.group(0).strip()                address_zip=re.findall(regex,htmlStr)                zip_match =[el for sub in address_zip for el in sub if el and el.split()[0] in zip_word]                                        if zip_match:                        zp=1                        zip_match=zip_match[0]                else:                    zip_match=None                    outputs.append(str(id1)+\t+str(id2)+\t+web+\t+ad+\t+word+\t+zp+\t+zip_match)                    if len(outputs)>=150:                        with open(filename,a) as  output:                output.write(\n.join(outputs)+\n)                del outputs[:]            except Exception,e:                print e                print traceback.print_exc()                with open(us_error.txt,a) as ded:                    ded.write(str(id1)+\t+web+\t+str(e)+\n)                           if outputs:            with open(filename,a) as  output:            output.write(\n.join(outputs)+\n)            del outputs[:]        infile = open(filename,'r')           ftp = ftplib.FTP(FTP_IP,FTP_USER ,FTP_PWD )                      ftp.storbinary('STOR '+FTP_LOCATION+str(filename), infile)             infile.close()                                         ftp.quit()    except Exception,e:        print e        print traceback.print_exc()        with open(us_error_process_error.txt,a) as ded:            ded.write( str(e))if __name__ == '__main__':    try:        access_string='DRIVER={SQL Server};SERVER='+SERVER+';DATABASE='+DATABASE+';UID='+UID+';PWD='+PWD        cnxn = pyodbc.connect(access_string,autocommit=True)        cursor = cnxn.cursor()        print sys.argv[1]+'\n'+sys.argv[2]        exec1=server_details[QUERY]        cursor.execute(exec1,(sys.argv[1],sys.argv[2]))        rows = cursor.fetchall()        cnxn.close()        ProcessFile(rows)        end_time_split=datetime.datetime.now()        elasped_time1=end_time_split-start_time_split        print Total time:+str(elasped_time1)        print done    except Exception,e:        print e        print traceback.print_exc()        with open(us_error_starting_process_error.txt,a) as ded:            ded.write(str(e))
URL pattern matching
python;regex