I've decided to write my first project in Python. I would like to hear some opinion from you.Description of the script:Generate Trivago URLs for 5 star hotels in specified city.Scrap these URLs to get prices.Save results as SQL queries in text fileExecute queries from text file to upload results to database.Settings:The date from which the script is going to check prices.Number of days to check.City to check.Name of file with results of the script.I wrote this script to learn how website scraping in Python works, and how to use SQL. I also wanted to learn something about object-oriented programming.__author__ = ''import datetimeimport pymysqlimport lxml.html as lhimport reimport sysfrom selenium import webdriverclass TrivagoPriceChecker():    from_year = ''    from_month = ''    from_day = ''    days_number = ''    city_id = ''    hotel_id = ''    result_file = ''    browser = webdriver.PhantomJS()    def __init__(self):        print(Trivago Price Checker ver 1.0)    def generate_url(self):        from_date = datetime.date(int(self.from_year), int(self.from_month), int(self.from_day))        to_date = datetime.date(int(self.from_year),int(self.from_month),int(self.from_day)) + datetime.timedelta(days=int(self.days_number))        url_list = []        while(from_date < to_date):            day_plus = from_date + datetime.timedelta(days=1)            url = 'http://www.trivago.pl/?aDateRange%5Barr%5D=' + str(from_date) + '&aDateRange%5Bdep%5D=' + str(day_plus) + '&iRoomType=7&iPathId=' + str(self.city_id) + '&iGeoDistanceItem=' + str(self.hotel_id) + '&iViewType=0&bIsSeoPage=false&bIsSitemap=false&'            url_list.append(url)            from_date += datetime.timedelta(days=1)        return url_list    def get_hotel_price(self, hotel_url):        self.browser.get(hotel_url)        content = self.browser.page_source        website = lh.fromstring(content)        for price in website.xpath('//*[@id=js_item_' + str(self.hotel_id) + ']/div[1]/div[2]/div[2]/strong[2]'):            return price.text    def save_result(self):        date = datetime.date(int(self.from_year), int(self.from_month), int(self.from_day))        file = open(self.result_file, a)        counter = 1        for result in self.generate_url():            try:                price = self.get_hotel_price(result).strip()                price = re.sub('[^0-9]', '', price)                sql_query = INSERT INTO prices (hotel, city, adate, price) VALUES(' + str(self.hotel_id) +',' + str(self.city_id) + ',' + str(date) + ',' + str(price) + ');                file.write(sql_query)                file.write('\n')                print('[' + str(counter) + '/' + str(self.days_number) + '] Hotel ID: ' + str(self.hotel_id))            except AttributeError:                print('[' + str(counter) + '/' + str(self.days_number) + '] Hotel ID: ' + str(self.hotel_id) + ' Sold out!')            counter += 1            date = date + datetime.timedelta(days=1)        file.close()poland = {        poznan: {city_id: 86470, hotel_id: [1711505, 163780, 932461, 1164703]},        warszawa: {city_id: 86484, hotel_id: [1503333, 93311, 93181, 93268, 106958, 106956, 127649, 106801, 107386, 93245, 154078, 107032]},        sopot: {city_id: 95266, hotel_id: [228481, 164126, 922891]},        gdansk: {city_id: 86490, hotel_id: [102961, 1008151, 102944, 1503323]},        krakow: {city_id: 86473, hotel_id: [931575, 925925, 102937, 148894, 125181, 930571, 114768, 125763, 106926, 102947, 131257]},        wroclaw: {city_id: 86485, hotel_id: [122767, 123690, 2873646, 1300328, 1511989, 121719]},        ilawa: {city_id: 110111, hotel_id: [2728378]},        bydgoszcz: {city_id: 86475, hotel_id: [936931]},        kolobrzeg: {city_id: 114376, hotel_id: [1288624, 1393804, 3185658, 1217228]},        mikolajki: {city_id: 110236, hotel_id: [2873760]},        rzeszow: {city_id: 86472, hotel_id: [2591078]},        zakopane: {city_id: 112161, hotel_id: [408841, 1828491, 320661]},        ostroda: {city_id: 110301, hotel_id: [966969]},        czeladz: {city_id: 458329, hotel_id: [2030401]},        gietrzwald: {city_id: 110071, hotel_id: [2733447]},        krynica_zdroj: {city_id: 111696, hotel_id: [1226658]},        tychy: {city_id: 86502, hotel_id: [164039]},        kielce: {city_id: 86471, hotel_id: [1941137]},        miedziana_gora: {city_id: 470673, hotel_id: [2175600]},        brojce: {city_id: 467917, hotel_id: [412116]},        ustka: {city_id: 93762, hotel_id: [3082744]},        lublin: {city_id: 86481, hotel_id: [3083850]},        choczewo: {city_id: 113541, hotel_id: [3135678]},        dziwnow: {city_id: 114306, hotel_id: [3213582]},        ustron: {city_id: 114126, hotel_id: [966089]},        szczawnica: {city_id: 112051, hotel_id: [1259175]}}def check_city(from_year, from_month, from_day, days_number, city, result_file):    worker = TrivagoPriceChecker()    worker.from_year = from_year    worker.from_month = from_month    worker.from_day = from_day    worker.days_number = days_number    worker.result_file = result_file    if city in poland:        worker.city_id = poland[city][city_id]        print(worker.city_id)        for x in poland[city][hotel_id]:            worker.hotel_id = x            worker.save_result()    else:        print(City not found!)        exit()def export_results(db_host, db_port, db_user, db_password, db_name, query_file):    connection = pymysql.connect(host=str(db_host), port=db_port, user=str(db_user), passwd=str(db_password), db=str(db_name))    query = connection.cursor()    file = open(query_file,r)    progress = 0    for line in file:        try:            query.execute(line)            progress += 1            print(progress)        except:            pass    connection.commit()    file.close()    connection.close()if __name__ == __main__:    if len(sys.argv) == 7:        check_city(str(sys.argv[1]), str(sys.argv[2]), str(sys.argv[3]), str(sys.argv[4]), str(sys.argv[5]), str(sys.argv[6]))    else:        print(Example usage: main.py 2015 02 01 30 sopot sopot.txt)
Trivago hotels price checker
python;beginner;web scraping
Style and bad pracitcesEven ignoring Caridorc's point about splitting the imports, they aren't alphabetical. They should be.Wrap your lines! There is no reason to even consider a 265-character long line.Stop converting: you're going backa and forth with datatypes like no tomorrow. There's no reason for from_year to ever be a string if it's required to be a valid integer. Convert once and keep it so.You even dostr(sys.argv[1]), str(sys.argv[2]), str(sys.argv[3]), str(sys.argv[4]), str(sys.argv[5]), str(sys.argv[6])despite each argument already being a string! Really you should be doingint(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5], sys.argv[6]!Plurals. hotel_ids, not hotel_id. generate_urls, not generate_url.__init__ is your initializer. Let it initialize. Don't do this monstrosity:worker = TrivagoPriceChecker()worker.from_year = from_yearworker.from_month = from_monthworker.from_day = from_dayworker.days_number = days_numberworker.result_file = result_fileIt should beworker = TrivagoPriceChecker(from_year, from_month, from_day, days_number, result_file)Globals!This is important enough to get its own section. When you doclass TrivagoPriceChecker():    from_year = ''    from_month = ''    from_day = ''    days_number = ''    city_id = ''    hotel_id = ''    result_file = ''    browser = webdriver.PhantomJS()you make a particularly evil type of global: one that can be shadowed at any time by any assignment statement to it through self. You don't need these defaults at all: just make a proper __init__. Even if you did want defaults like these, you'd want to set them in __init__.Even if you think you want a global browser, you probably want it shared manually for extensibility in the future.Programming with graceFiles and pymysql support with. Useconnection = pymysql.connect(host=db_host, port=db_port, user=db_user, passwd=db_password, db=db_name)with connection as query, open(query_file, r) as file:    progress = 0    for line in file:        try:            query.execute(line)            progress += 1            print(progress)        except:            passAlso consider enumerateconnection = pymysql.connect(host=db_host, port=db_port, user=db_user, passwd=db_password, db=db_name)with connection as query, open(query_file, r) as file:    for progress, line in enumerate(file):        try:            query.execute(line)            print(progress)        except:            passAlso, reduce the area of the try and the number of things it catches. I see that stupid exceptions are thrown, but you should at least avoid catching BaseException (which includes things like KeyboardInterrupt and SystemExit) and just doconnection = pymysql.connect(host=db_host, port=db_port, user=db_user, passwd=db_password, db=db_name)with connection as query, open(query_file, r) as file:    for progress, line in enumerate(file):        try:            query.execute(line)        except Exception:            pass        print(progress)For that matter: exceptionsDon't ever do this:if city in poland:    # codeelse:    print(City not found!)    exit()Not only are you hiding the error by separating it from the condition that triggers it, you're calling exit directly! You're even calling the wrong exit! Do:if city not in poland:    raise ValueError(City not found!)# codeWhen you doexcept AttributeError:there's a large block of code that could be at fault. Usetry:    # small piece of codeexcept AttributeError:    # ...else:    # rest of codeto narrow this down to something maintainable. In this case, you shouldn't even be using try ... except.DRY, not DRY DRYYou have, after prior simplification:from_date = datetime.date(self.from_year, self.from_month, self.from_day)to_date = datetime.date(self.from_year, self.from_month, self.from_day) + datetime.timedelta(days=self.days_number)Why? This should just befrom_date = datetime.date(self.from_year, self.from_month, self.from_day)to_date = from_date + datetime.timedelta(days=self.days_number)In save_result you again dodate = datetime.date(self.from_year, self.from_month, self.from_day)This is silly; you call this a lot. In __init__, just initialize self.from_date and self.to_date and drop the other nonsense.Don'tdate = date + datetime.timedelta(days=1)usedate += datetime.timedelta(days=1)You do this just above; surely you know of it.Use formatting instead of str(a) + 'xxx' + str(b) + 'xxx' + ...:sql_query = (    INSERT INTO prices (hotel, city, adate, price)    VALUES('{self.hotel_id}'','{self.city_id}','{date}','{price}');).format(self=self, date=date, price=price)Which brings me to...SQL INJECTION!You should escape. The simple way is justsql_query = (    INSERT INTO prices (hotel, city, adate, price)    VALUES('{hotel_id}'','{city_id}','{date}','{price}');).format(    hotel_id=pymysql.escape_string(self.hotel_id),    city_id=pymysql.escape_string(self.city_id),    date=date,    price=price)In reality, any executing query should use better mechanisms (like interpolating directly with cursor.execute) but you're writing to a file so this isn't as easy.And finallyYou pass hotel_id as a parameter to save_result by setting an instance variable. This is ugly; just pass a parameter. city_id should probably passed the same way for symmetry.I've done a little more cleaning and came up with this.__author__ = 'Mateusz Ostaszewski'import datetimeimport lxml.html as lhimport reimport sysimport pymysqlfrom selenium import webdriverclass  TrivagoPriceChecker:    def __init__(self, browser, year, month, day, days_number, result_file):        print(Trivago Price Checker ver 1.0)        self.browser = browser        self.from_date = datetime.date(year, month, day)        self.days_number = days_number        self.result_file = result_file    def generate_urls(self, city_id, hotel_id):        url = (            http://www.trivago.pl/            ?aDateRange%5Barr%5D={}            &aDateRange%5Bdep%5D={}            &iRoomType=7&iPathId={}            &iGeoDistanceItem={}            &iViewType=0            &bIsSeoPage=false            &bIsSitemap=false        )        def make_url(day_num):            day = self.from_date + datetime.timedelta(days=day_num)            return day, url.format(day, day, city_id, hotel_id)        return [make_url(day) for day in range(self.days_number)]    def get_hotel_price(self, hotel_id, hotel_url):        self.browser.get(hotel_url)        content = self.browser.page_source        website = lh.fromstring(content)        # Get first if exists, otherwise return None        for price in website.xpath('//*[@id=js_item_{}]/div[1]/div[2]/div[2]/strong[2]'.format(hotel_id)):            return price.text    def save_result(self, city_id, hotel_id):        def esc(x):            return pymysql.escape_string(str(x))        query = INSERT INTO prices (hotel, city, adate, price) VALUES('{}','{}','{{}}','{{}}');\n        query = query.format(esc(hotel_id), esc(city_id))        with open(self.result_file, a) as file:            for day, result in self.generate_urls(city_id, hotel_id):                price = self.get_hotel_price(hotel_id, result).strip()                if price is not None:                    price = re.sub('[^0-9]', '', price)                    file.write(query.format(esc(day), esc(price)))                soldout =  Sold out! if price is None else                 print('[{}/{}] Hotel ID: {}{}'.format(day, self.days_number, hotel_id, soldout))poland = {    poznan: {city_id: 86470, hotel_ids: [1711505, 163780, 932461, 1164703]},    warszawa: {city_id: 86484, hotel_ids: [1503333, 93311, 93181, 93268, 106958, 106956,                                                 127649, 106801, 107386, 93245, 154078, 107032]},    sopot: {city_id: 95266, hotel_ids: [228481, 164126, 922891]},    gdansk: {city_id: 86490, hotel_ids: [102961, 1008151, 102944, 1503323]},    krakow: {city_id: 86473, hotel_ids: [931575, 925925, 102937, 148894, 125181, 930571,                                               114768, 125763, 106926, 102947, 131257]},    wroclaw: {city_id: 86485, hotel_ids: [122767, 123690, 2873646, 1300328, 1511989, 121719]},    ilawa: {city_id: 110111, hotel_ids: [2728378]},    bydgoszcz: {city_id: 86475, hotel_ids: [936931]},    kolobrzeg: {city_id: 114376, hotel_ids: [1288624, 1393804, 3185658, 1217228]},    mikolajki: {city_id: 110236, hotel_ids: [2873760]},    rzeszow: {city_id: 86472, hotel_ids: [2591078]},    zakopane: {city_id: 112161, hotel_ids: [408841, 1828491, 320661]},    ostroda: {city_id: 110301, hotel_ids: [966969]},    czeladz: {city_id: 458329, hotel_ids: [2030401]},    gietrzwald: {city_id: 110071, hotel_ids: [2733447]},    krynica_zdroj: {city_id: 111696, hotel_ids: [1226658]},    tychy: {city_id: 86502, hotel_ids: [164039]},    kielce: {city_id: 86471, hotel_ids: [1941137]},    miedziana_gora: {city_id: 470673, hotel_ids: [2175600]},    brojce: {city_id: 467917, hotel_ids: [412116]},    ustka: {city_id: 93762, hotel_ids: [3082744]},    lublin: {city_id: 86481, hotel_ids: [3083850]},    choczewo: {city_id: 113541, hotel_ids: [3135678]},    dziwnow: {city_id: 114306, hotel_ids: [3213582]},    ustron: {city_id: 114126, hotel_ids: [966089]},    szczawnica: {city_id: 112051, hotel_ids: [1259175]}}def check_city(year, month, day, days_number, city, result_file):    if city not in poland:        raise ValueError(City not found!)    browser = webdriver.PhantomJS()    worker = TrivagoPriceChecker(browser, year, month, day, days_number, result_file)    city_id = poland[city][city_id]    print(city_id)    for hotel_id in poland[city][hotel_ids]:        worker.save_result(city_id, hotel_id)def export_results(db_host, db_port, db_user, db_password, db_name, query_file):    connection = pymysql.connect(host=db_host, port=db_port, user=db_user, passwd=db_password, db=db_name)    with connection as query, open(query_file, r) as file:        for progress, line in enumerate(file):            try:                query.execute(line)            except Exception:                pass            print(progress)def main():    try:        _, year, month, day, days_number, city, result_file = sys.argv    except ValueError:        raise SystemExit(Example usage: main.py 2015 02 01 30 sopot sopot.txt)    check_city(int(year), int(month), int(day), int(days_number), city, result_file)if __name__ == __main__:    main()