<jupyter_start><jupyter_text>data.drop_duplicates()  的作用是直接抛弃重复值。 但是要考虑到板块的数据有很多缺失值，因为drop_duplicates是默认抛弃第二条重复的值 若两条数据，前面的cjdnajia，cjxiaoqu，cilouceng是重复的，但是第一条的bankuai为null值，第二条是有数据的，则会默认抛弃第二条有效信息 为了避免这种情况，我们要把null值的那条全部放到后面去。<jupyter_code>(data.duplicated(subset=['cjdanjia','cjxiaoqu','cjlouceng','bankuai'])).sum()
data.sort_values(by='bankuai',inplace=True)
data.drop_duplicates(subset=['cjdanjia','cjxiaoqu','cjlouceng'],inplace=True)
#数据类型转换，数据异常值处理，数据离散化处理
data.head()
(~data.cjdanjia.str.contains('元/平')).sum()
data.cjdanjia.map(lambda x:round(float(x.replace('元/平',''))/10000,2))
data.cjdanjia.str.replace('元/平','').astype(np.float32).map(lambda x:round(x/10000,2))
data=data.assign(cjdanjia=data.cjdanjia.str.replace('元/平','').astype(np.float32).map(lambda x:round(x/10000,2)))
data['cjdanjia']=data.cjdanjia.str.replace('元/平','').astype(np.float32).map(lambda x:round(x/10000,2))
data.head()
data.cjdanjia.min()
data.cjdanjia.max()
data=data[data.cjdanjia>0]
data.cjdanjia.min()
data=data[data.cjdanjia>0.5]
data.cjdanjia.min()
len(data)
bins=[0,1,2,3,4,5,7,9,11,13,15]
pd.cut(data.cjdanjia,bins)
pd.cut(data.cjdanjia,bins).value_counts()
pd.cut(data.cjdanjia,bins).value_counts().plot()
pd.cut(data.cjdanjia,bins).value_counts().plot.bar(rot=20)
pd.cut(data.cjdanjia,bins).value_counts().plot.pie(figsize=(8,8))<jupyter_output><empty_output><jupyter_text>字符串处理<jupyter_code>data.head()
(data.cjlouceng.str.split('/').map(len)!=3).sum()
data.cjlouceng
data.cjlouceng.map(lambda x:x.split('/')[0])
data['chaoxiang']=data.cjlouceng.map(lambda x:x.split('/')[0])
data
data['louceng']=data.cjlouceng.map(lambda x:x.split('/')[1])
data
data.louceng.unique()
#还有未知？还有空字符串？对于这部分数据我们可以处理一下。
data[data.louceng=='']
data[data.louceng=='未知']
data=data[(data.louceng!='未知')&(data.louceng!='')]
data
pd.get_dummies(data.louceng)
data.join(pd.get_dummies(data.louceng))
data
pd.get_dummies(data.louceng).sum()
pd.get_dummies(data.louceng).sum().plot.bar()
(pd.get_dummies(data.louceng).sum()).to_csv('loucengfenbu3.csv',encoding='utf_8_sig')<jupyter_output><empty_output><jupyter_text>分组运算与布尔过滤与透视表<jupyter_code>data.head()
data['cjshijian']=data.cjshijian.map(lambda x:x.split('：')[1])
data['year']=data.cjshijian.map(lambda x:x.split('-')[0])
data.groupby(['year','xingming'])['xingming'].value_counts()
data.groupby(['year','xingming'])['xingming'].count()
data.groupby(['year','xingming'])['xingming'].count().to_csv('butongnianfenjingjiren.csv',encoding='utf_8_sig')
data.groupby(['year','mendian'])['mendian'].count()
data_group=data.groupby(['xingming','congyenianxian'])['cjzongjia'].sum()
data_group[data_group>10000]
data_group[data_group>10000].plot()
data_1w=data.pivot_table('cjzongjia',index='xingming',columns='congyenianxian',aggfunc=sum)
data_1w
(data_1w>10000).sum()
plt.rcParams['font.sans-serif']=['SimHei']  #用来显示中文标签
(data_1w>10000).sum().plot.bar()
plt.rcParams['font.sans-serif']=['simhei']  #用来显示中文标签

(data_1w>10000).sum().plot.bar()
import matplotlib
matplotlib.matplotlib_fname()
((data_1w>10000).sum()).to_csv('nianxianfenbu1.csv',encoding='utf_8_sig')<jupyter_output><empty_output><jupyter_text>自己的分析<jupyter_code>data.head()
data.head()
data.groupby('year')['cjdanjia'].count()
data.groupby('year')['cjdanjia'].count().to_csv('chengjiaoliang_year.csv',encoding='utf_8_sig')
data.groupby('year')['cjdanjia'].mean()
(data.groupby('year')['cjdanjia'].mean()).to_csv('danjiajunzhi_year.csv',encoding='utf_8_sig')
data.groupby('year')['cjzongjia'].mean()
(data.groupby('year')['cjzongjia'].mean()).to_csv('chengjiaozongjia_year.csv',encoding='utf_8_sig')
data.groupby('bankuai')['cjdanjia'].count()
(data.groupby('bankuai')['cjdanjia'].count()).to_csv('butongbankuaichengjiao.csv',encoding='utf_8_sig')
bins=[0,2,4,6,8,12,15]
fenbu=pd.cut(data['cjdanjia'],bins)
data['cjdanjiafenbu']=fenbu
data.groupby(['bankuai','cjdanjiafenbu'])['cjdanjiafenbu'].count()
(data.groupby(['bankuai','cjdanjiafenbu'])['cjdanjiafenbu'].count()).to_csv('cjdanjiafenbu.csv',encoding='utf_8_sig')
data.groupby(['year','mendian'])['mendian'].count()
(data.groupby(['year','mendian'])['mendian'].count()).to_csv('butongnianxianmendian.csv',encoding='utf_8_sig')
data.groupby('zhiwei')['zhiwei'].count()
(data.groupby('zhiwei')['zhiwei'].count()).to_csv('zhiwei.csv',encoding='utf_8_sig')
bins=[0,2,4,6,8,12,15]
fenbu=pd.cut(data['cjdanjia'],bins)
data.groupby(['year','cjdanjiafenbu'])['cjdanjiafenbu'].count()
(data.groupby(['year','cjdanjiafenbu'])['cjdanjiafenbu'].count()).to_csv('xiaofeizhejiage.csv',encoding='utf_8_sig')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2337063857801185, 'fraction_numerical': 0.027210884353741496, 'mean_word_length': 13.655948553054662, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '476929', 'license': 'MIT', 'n_tokens_mistral': 2216, 'n_tokens_neox': 2179, 'n_words': 100, 'path': 'utils/lianjia_data_mining.ipynb', 'repo_name': 'tenkeyseven/DataMining_Price_Prediction', 'size': '4557'}