<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author: Oros
# Contributors :
#  purwowd
#  puyoulu
#  1kali2kali
#  petterreinholdtsen
#  nicoeg
#  dspinellis
#  fdl <<EMAIL>>
#  lapolis
# 2020-08-15
# License : CC0 1.0 Universal

"""
This program shows you IMSI numbers of cellphones around you.


/! This program was made to understand how GSM network work. Not for bad hacking !
"""

import ctypes
import json
from optparse import OptionParser
import datetime
import io
import socket

imsitracker = None


class tracker:
    imsistate = {}
    # phones
    imsis = []  # [IMSI,...]
    tmsis = {}  # {TMSI:IMSI,...}
    nb_IMSI = 0  # count the number of IMSI

    mcc = ""
    mnc = ""
    lac = ""
    cell = ""
    country = ""
    brand = ""
    operator = ""

    # in minutes
    purgeTimer = 10  # default 10 min

    show_all_tmsi = False
    mcc_codes = None
    sqlite_con = None
    mysql_con = None
    mysql_cur = None
    textfilePath = None
    output_function = None

    def __init__(self):
        self.load_mcc_codes()
        self.track_this_imsi("")
        self.output_function = self.output

    def set_output_function(self, new_output_function):
        # New output function need this field :
        # cpt, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell, timestamp, packet=None
        self.output_function = new_output_function

    def track_this_imsi(self, imsi_to_track):
        self.imsi_to_track = imsi_to_track
        self.imsi_to_track_len = len(imsi_to_track)

    # return something like '0xd9605460'
    def str_tmsi(self, tmsi):
        if tmsi != "":
            new_tmsi = "0x"
            for a in tmsi:
                c = hex(a)
                if len(c) == 4:
                    new_tmsi += str(c[2]) + str(c[3])
                else:
                    new_tmsi += "0" + str(c[2])
            return new_tmsi
        else:
            return ""

    def decode_imsi(self, imsi):
        new_imsi = ''
        for a in imsi:
            c = hex(a)
            if len(c) == 4:
                new_imsi += str(c[3]) + str(c[2])
            else:
                new_imsi += str(c[2]) + "0"

        mcc = new_imsi[1:4]
        mnc = new_imsi[4:6]
        return new_imsi, mcc, mnc

    # return something like
    # '208 20 1752XXXXXX', 'France', 'Bouygues', 'Bouygues Telecom'
    def str_imsi(self, imsi, packet=""):
        new_imsi, mcc, mnc = self.decode_imsi(imsi)
        country = ""
        brand = ""
        operator = ""

        if mcc in self.mcc_codes:
            if mnc in self.mcc_codes[mcc]:
                brand, operator, country, _ = self.mcc_codes[mcc][mnc]
                new_imsi = mcc + " " + mnc + " " + new_imsi[6:]
            elif mnc + new_imsi[6:7] in self.mcc_codes[mcc]:
                mnc += new_imsi[6:7]
                brand, operator, country, _ = self.mcc_codes[mcc][mnc]
                new_imsi = mcc + " " + mnc + " " + new_imsi[7:]

        else:
            country = f"Unknown MCC {mcc}"
            brand = f"Unknown MNC {mnc}"
            operator = f"Unknown MNC {mnc}"
            new_imsi = f"{mcc} {mnc} {new_imsi[6:]}"

        try:
            return new_imsi, country, brand, operator
        except Exception:
            # m = ""
            print("Error", packet, new_imsi, country, brand, operator)
        return "", "", "", ""

    def load_mcc_codes(self):
        # mcc codes form https://en.wikipedia.org/wiki/Mobile_Network_Code
        with io.open('mcc-mnc/mcc_codes.json', 'r', encoding='utf8') as file:
            self.mcc_codes = json.load(file)

    def current_cell(self, mcc, mnc, lac, cell):
        brand = ""
        operator = ""
        country = ""
        if mcc in self.mcc_codes and mnc in self.mcc_codes[mcc]:
            brand, operator, country, _ = self.mcc_codes[mcc][mnc]
        else:
            country = f"Unknown MCC {mcc}"
            brand = f"Unknown MNC {mnc}"
            operator = f"Unknown MNC {mnc}"
        self.mcc = str(mcc)
        self.mnc = str(mnc)
        self.lac = str(lac)
        self.cell = str(cell)
        self.country = country
        self.brand = brand
        self.operator = operator

    def sqlite_file(self, filename):
        import sqlite3  # Avoid pulling in sqlite3 when not saving
        print("Saving to SQLite database in %s" % filename)
        self.sqlite_con = sqlite3.connect(filename)
        self.sqlite_con.text_factory = str
        # FIXME Figure out proper SQL type for each attribute
        self.sqlite_con.execute("CREATE TABLE IF NOT EXISTS observations(stamp datetime, tmsi1 text, tmsi2 text, imsi text, imsicountry text, imsibrand text, imsioperator text, mcc integer, mnc integer, lac integer, cell integer);")

    def text_file(self, filename):
        txt = open(filename, "w")
        txt.write("stamp, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell\n")
        txt.close()
        self.textfilePath = filename

    def mysql_file(self):
        import os.path
        if os.path.isfile('.env'):
            import MySQLdb as mdb
            from decouple import config
            self.mysql_con = mdb.connect(config("MYSQL_HOST"), config("MYSQL_USER"), config("MYSQL_PASSWORD"), config("MYSQL_DB"))
            self.mysql_cur = self.mysql_con.cursor()
            # Check MySQL connection
            if self.mysql_cur:
                print("mysql connection is success :)")
            else:
                print("mysql connection is failed!")
                exit()
        else:
            print("create file .env first")
            exit()

    def output(self, cpt, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell, now, packet=None):
        print(f"{str(cpt):7s} ; {tmsi1:10s} ; {tmsi2:10s} ; {imsi:17s} ; {imsicountry:14s} ; {imsibrand:10s} ; {imsioperator:21s} ; {str(mcc):4s} ; {str(mnc):5s} ; {str(lac):6s} ; {str(cell):6s} ; {now.isoformat():s}")

    def pfields(self, cpt, tmsi1, tmsi2, imsi, mcc, mnc, lac, cell, packet=None):
        imsicountry = ""
        imsibrand = ""
        imsioperator = ""
        if imsi:
            imsi, imsicountry, imsibrand, imsioperator = self.str_imsi(imsi, packet)
        else:
            imsi = ""
        now = datetime.datetime.now()
        self.output_function(cpt, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell, now, packet)

        if self.textfilePath:
            now = datetime.datetime.now()
            txt = open(self.textfilePath, "a")
            txt.write(f"{str(now)}, {tmsi1}, {tmsi2}, {imsi}, {imsicountry}, {imsibrand}, {imsioperator}, {mcc}, {mnc}, {lac}, {cell}\n")
            txt.close()

        if tmsi1 == "":
            tmsi1 = None
        if tmsi2 == "":
            tmsi2 = None

        if self.sqlite_con:
            self.sqlite_con.execute(
               u"INSERT INTO observations (stamp, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
               (now, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell)
            )
            self.sqlite_con.commit()
            pass

        if self.mysql_cur:
            print("saving data to db...")
            # Example query
            query = ("INSERT INTO `imsi` (`tmsi1`, `tmsi2`, `imsi`,`mcc`, `mnc`, `lac`, `cell_id`, `stamp`, `deviceid`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
            arg = (tmsi1, tmsi2, imsi, mcc, mnc, lac, cell, now, "rtl")
            self.mysql_cur.execute(query, arg)
            self.mysql_con.commit()

    def header(self):
        print(f"{'Nb IMSI':7s} ; {'TMSI-1':10s} ; {'TMSI-2':10s} ; {'IMSI':17s} ; {'country':14s} ; {'brand':10s} ; {'operator':21s} ; {'MCC':4s} ; {'MNC':5s} ; {'LAC':6s} ; {'CellId':6s} ; {'Timestamp':s}")

    def register_imsi(self, arfcn, imsi1="", imsi2="", tmsi1="", tmsi2="", p=""):
        do_print = False
        n = ''
        tmsi1 = self.str_tmsi(tmsi1)
        tmsi2 = self.str_tmsi(tmsi2)
        if imsi1:
            self.imsi_seen(imsi1, arfcn)
        if imsi2:
            self.imsi_seen(imsi2, arfcn)
        if imsi1 and (not self.imsi_to_track or imsi1[:self.imsi_to_track_len] == self.imsi_to_track):
            if imsi1 not in self.imsis:
                # new IMSI
                do_print = True
                self.imsis.append(imsi1)
                self.nb_IMSI += 1
                n = self.nb_IMSI
            if self.tmsis and tmsi1 and (tmsi1 not in self.tmsis or self.tmsis[tmsi1] != imsi1):
                # new TMSI to an ISMI
                do_print = True
                self.tmsis[tmsi1] = imsi1
            if self.tmsis and tmsi2 and (tmsi2 not in self.tmsis or self.tmsis[tmsi2] != imsi1):
                # new TMSI to an ISMI
                do_print = True
                self.tmsis[tmsi2] = imsi1

        if imsi2 and (not self.imsi_to_track or imsi2[:self.imsi_to_track_len] == self.imsi_to_track):<commit_msg>Fixing my fix - Unicode stuff<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author: Oros
# Contributors :
#  purwowd
#  puyoulu
#  1kali2kali
#  petterreinholdtsen
#  nicoeg
#  dspinellis
#  fdl <<EMAIL>>
#  lapolis
# 2020-08-15
# License : CC0 1.0 Universal

"""
This program shows you IMSI numbers of cellphones around you.


/! This program was made to understand how GSM network work. Not for bad hacking !
"""

import ctypes
import json
from optparse import OptionParser
import datetime
import io
import socket

imsitracker = None


class tracker:
    imsistate = {}
    # phones
    imsis = []  # [IMSI,...]
    tmsis = {}  # {TMSI:IMSI,...}
    nb_IMSI = 0  # count the number of IMSI

    mcc = ""
    mnc = ""
    lac = ""
    cell = ""
    country = ""
    brand = ""
    operator = ""

    # in minutes
    purgeTimer = 10  # default 10 min

    show_all_tmsi = False
    mcc_codes = None
    sqlite_con = None
    mysql_con = None
    mysql_cur = None
    textfilePath = None
    output_function = None

    def __init__(self):
        self.load_mcc_codes()
        self.track_this_imsi("")
        self.output_function = self.output

    def set_output_function(self, new_output_function):
        # New output function need this field :
        # cpt, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell, timestamp, packet=None
        self.output_function = new_output_function

    def track_this_imsi(self, imsi_to_track):
        self.imsi_to_track = imsi_to_track
        self.imsi_to_track_len = len(imsi_to_track)

    # return something like '0xd9605460'
    def str_tmsi(self, tmsi):
        if tmsi != "":
            new_tmsi = "0x"
            for a in tmsi:
                c = hex(a)
                if len(c) == 4:
                    new_tmsi += str(c[2]) + str(c[3])
                else:
                    new_tmsi += "0" + str(c[2])
            return new_tmsi
        else:
            return ""

    def decode_imsi(self, imsi):
        new_imsi = ''
        for a in imsi:
            c = hex(a)
            if len(c) == 4:
                new_imsi += str(c[3]) + str(c[2])
            else:
                new_imsi += str(c[2]) + "0"

        mcc = new_imsi[1:4]
        mnc = new_imsi[4:6]
        return new_imsi, mcc, mnc

    # return something like
    # '208 20 1752XXXXXX', 'France', 'Bouygues', 'Bouygues Telecom'
    def str_imsi(self, imsi, packet=""):
        new_imsi, mcc, mnc = self.decode_imsi(imsi)
        country = ""
        brand = ""
        operator = ""

        if mcc in self.mcc_codes:
            if mnc in self.mcc_codes[mcc]:
                brand, operator, country, _ = self.mcc_codes[mcc][mnc]
                new_imsi = f"{mcc} {mnc} {new_imsi[6:]}"
            elif mnc + new_imsi[6:7] in self.mcc_codes[mcc]:
                mnc += new_imsi[6:7]
                brand, operator, country, _ = self.mcc_codes[mcc][mnc]
                new_imsi = f"{mcc} {mnc} {new_imsi[7:]}"

        else:
            country = f"Unknown MCC {mcc}"
            brand = f"Unknown MNC {mnc}"
            operator = f"Unknown MNC {mnc}"
            new_imsi = f"{mcc} {mnc} {new_imsi[6:]}"

        try:
            return new_imsi, country, brand, operator
        except Exception:
            # m = ""
            print("Error", packet, new_imsi, country, brand, operator)
        return "", "", "", ""

    def load_mcc_codes(self):
        # mcc codes form https://en.wikipedia.org/wiki/Mobile_Network_Code
        with io.open('mcc-mnc/mcc_codes.json', 'r', encoding='utf8') as file:
            self.mcc_codes = json.load(file)

    def current_cell(self, mcc, mnc, lac, cell):
        brand = ""
        operator = ""
        country = ""
        if mcc in self.mcc_codes and mnc in self.mcc_codes[mcc]:
            brand, operator, country, _ = self.mcc_codes[mcc][mnc]
        else:
            country = f"Unknown MCC {mcc}"
            brand = f"Unknown MNC {mnc}"
            operator = f"Unknown MNC {mnc}"
        self.mcc = str(mcc)
        self.mnc = str(mnc)
        self.lac = str(lac)
        self.cell = str(cell)
        self.country = country
        self.brand = brand
        self.operator = operator

    def sqlite_file(self, filename):
        import sqlite3  # Avoid pulling in sqlite3 when not saving
        print("Saving to SQLite database in %s" % filename)
        self.sqlite_con = sqlite3.connect(filename)
        self.sqlite_con.text_factory = str
        # FIXME Figure out proper SQL type for each attribute
        self.sqlite_con.execute("CREATE TABLE IF NOT EXISTS observations(stamp datetime, tmsi1 text, tmsi2 text, imsi text, imsicountry text, imsibrand text, imsioperator text, mcc integer, mnc integer, lac integer, cell integer);")

    def text_file(self, filename):
        txt = open(filename, "w")
        txt.write("stamp, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell\n")
        txt.close()
        self.textfilePath = filename

    def mysql_file(self):
        import os.path
        if os.path.isfile('.env'):
            import MySQLdb as mdb
            from decouple import config
            self.mysql_con = mdb.connect(config("MYSQL_HOST"), config("MYSQL_USER"), config("MYSQL_PASSWORD"), config("MYSQL_DB"))
            self.mysql_cur = self.mysql_con.cursor()
            # Check MySQL connection
            if self.mysql_cur:
                print("mysql connection is success :)")
            else:
                print("mysql connection is failed!")
                exit()
        else:
            print("create file .env first")
            exit()

    def output(self, cpt, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell, now, packet=None):
        print(f"{str(cpt):7s} ; {tmsi1:10s} ; {tmsi2:10s} ; {imsi:17s} ; {imsicountry:16s} ; {imsibrand:14s} ; {imsioperator:21s} ; {str(mcc):4s} ; {str(mnc):5s} ; {str(lac):6s} ; {str(cell):6s} ; {now.isoformat():s}")

    def pfields(self, cpt, tmsi1, tmsi2, imsi, mcc, mnc, lac, cell, packet=None):
        imsicountry = ""
        imsibrand = ""
        imsioperator = ""
        if imsi:
            imsi, imsicountry, imsibrand, imsioperator = self.str_imsi(imsi, packet)
        else:
            imsi = ""
        now = datetime.datetime.now()
        self.output_function(cpt, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell, now, packet)

        if self.textfilePath:
            now = datetime.datetime.now()
            txt = open(self.textfilePath, "a")
            txt.write(f"{str(now)}, {tmsi1}, {tmsi2}, {imsi}, {imsicountry}, {imsibrand}, {imsioperator}, {mcc}, {mnc}, {lac}, {cell}\n")
            txt.close()

        if tmsi1 == "":
            tmsi1 = None
        if tmsi2 == "":
            tmsi2 = None

        if self.sqlite_con:
            self.sqlite_con.execute(
               u"INSERT INTO observations (stamp, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
               (now, tmsi1, tmsi2, imsi, imsicountry, imsibrand, imsioperator, mcc, mnc, lac, cell)
            )
            self.sqlite_con.commit()
            pass

        if self.mysql_cur:
            print("saving data to db...")
            # Example query
            query = ("INSERT INTO `imsi` (`tmsi1`, `tmsi2`, `imsi`,`mcc`, `mnc`, `lac`, `cell_id`, `stamp`, `deviceid`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)")
            arg = (tmsi1, tmsi2, imsi, mcc, mnc, lac, cell, now, "rtl")
            self.mysql_cur.execute(query, arg)
            self.mysql_con.commit()

    def header(self):
        print(f"{'Nb IMSI':7s} ; {'TMSI-1':10s} ; {'TMSI-2':10s} ; {'IMSI':17s} ; {'country':16s} ; {'brand':14s} ; {'operator':21s} ; {'MCC':4s} ; {'MNC':5s} ; {'LAC':6s} ; {'CellId':6s} ; {'Timestamp':s}")

    def register_imsi(self, arfcn, imsi1="", imsi2="", tmsi1="", tmsi2="", p=""):
        do_print = False
        n = ''
        tmsi1 = self.str_tmsi(tmsi1)
        tmsi2 = self.str_tmsi(tmsi2)
        if imsi1:
            self.imsi_seen(imsi1, arfcn)
        if imsi2:
            self.imsi_seen(imsi2, arfcn)
        if imsi1 and (not self.imsi_to_track or imsi1[:self.imsi_to_track_len] == self.imsi_to_track):
            if imsi1 not in self.imsis:
                # new IMSI
                do_print = True
                self.imsis.append(imsi1)
                self.nb_IMSI += 1
                n = self.nb_IMSI
            if self.tmsis and tmsi1 and (tmsi1 not in self.tmsis or self.tmsis[tmsi1] != imsi1):
                # new TMSI to an ISMI
                do_print = True
                self.tmsis[tmsi1] = imsi1
            if self.tmsis and tmsi2 and (tmsi2 not in self.tmsis or self.tmsis[tmsi2] != imsi1):
                # new TMSI to an ISMI
                do_print = True
                self.tmsis[tmsi2] = imsi1

        if imsi2 and (not self.imsi_to_track or imsi2[:self.imsi_to_track_len] == self.imsi_to_track):
{'fraction_non_alphanumeric': 0.1316633488166335, 'fraction_numerical': 0.017474010174740102, 'mean_word_length': 2.035924122880645, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3790808', 'n_tokens_mistral': 6656, 'n_tokens_neox': 6436, 'n_words': 1493}