<jupyter_start><jupyter_text>1. rawdata<jupyter_code>df = pd.read_excel('data/network_paper_rank.xlsx',index_col=0)
df.shape
df.sample()<jupyter_output><empty_output><jupyter_text>2.get citation from semantic scholar<jupyter_code>def get_full_record(doi,offset=0,limit=1000):
    url = f'https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}'\
            f'/citations?fields=year,citationCount,fieldsOfStudy&offset={offset}&limit={limit}'
    response = requests.get(url)
    rst = response.json()
    while len(response.json()['data'])==1000:
        offset += 1000
        url = f'https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}'\
        f'/citations?fields=year,citationCount,fieldsOfStudy&offset={offset}&limit={limit}'
        response  = requests.get(url)
        rst['data'].extend(response.json()['data'])
    return rst
lyst1 = []
lyst2 = []
for i in range(len(df)):
    doi = df.loc[i,'doi']
    rst = get_full_record(doi)
    #print(f"论文名:{df.loc[i,'title']},被引量{len(rst['data'])}")
    lyst1.append(len(rst['data']))
    lyst2.append(rst)
    
df.loc[:,'citation_ss'] = lyst1
df.loc[:,'citation_per_year'] = lyst2
df.sample()
# #an offline version due 2020.03.16
# df = pd.read_json('data/PR_network_science_paper.json',lines=True)<jupyter_output><empty_output><jupyter_text>3. longterm citation prediction<jupyter_code># fit function

# M measures the average number of references each new paper contains
M = 20
def fi(x):
    return norm.cdf(x)

def func(x,lam,u,sigma):
    return M*(np.exp(lam*fi((np.log(x) - u)/sigma))-1)

def infinite_citation(lam):
    return M*(np.exp(lam) - 1)
df.sample()
df = df.reset_index(drop=True)
df[['title','year']]
for idx,row in df.iterrows():
    nums = {}
    for item in  df.loc[idx,'citation_per_year']['data']:
        if item['citingPaper']['year']:
            nums[item['citingPaper']['year']] = nums.setdefault(item['citingPaper']['year'],0) + 1
    
    nums = {k:v for k,v in nums.items() if k >= df.loc[idx,'year']}
    x = np.arange(min(nums),2022)
    y = [nums[l] if l in nums else 0 for l in x]
    
    x = np.arange(1,2022-min(nums)+1)
    y_cumsum = np.cumsum(y)
#     print('title:',df.loc[idx,'title'])
#     print('citation per year:',nums)
#     print('x:',x)
#     print('y:',y)
#     print('y_cumsum:',y_cumsum)
    
    # at least 3 data points needed
    if len(x)>2:
        coes,cov = curve_fit(func,x,y_cumsum)
        #print('lambda,u,sigma:',coes)
        lam = coes[0]
        final = infinite_citation(lam)
        #print('final:',final)
        df.loc[idx,'infinite_citation'] = final
    df.loc[idx,'first_two_year_citation'] = y[0]+y[1]
    #print('\n')
# infinite_citation column added
df.sample()
# paper with num of data points < 3
ratio = 0
i = 0
for idx,row in df.iterrows():
    if not np.isnan(row['infinite_citation']):
        ratio += row['infinite_citation']/row['first_two_year_citation']
        i += 1
        
ave_ratio =  ratio/i

for idx,row in df.iterrows():
    if np.isnan(row['infinite_citation']):
        df.loc[idx,'infinite_citation'] = df.loc[idx,'first_two_year_citation']*ave_ratio
df[['title','year','infinite_citation']].to_csv('rst.csv',index=0,encoding='utf-8')<jupyter_output><empty_output><jupyter_text>4. case<jupyter_code>def plot_longterm_citation(x,y,func,coes):
    X = np.arange(1,50)
    plt.figure(figsize=(9,6))
    plt.plot(x,y,'*',label='record')
    plt.plot(X,func(X,coes[0],coes[1],coes[2]),alpha=0.5,label='predict')
    plt.xlabel('year',fontsize=15)
    plt.ylabel('citation',fontsize=15)
    plt.legend(fontsize=15)
    return 
idx = 24
nums = {}
for item in  df.loc[idx,'citation_per_year']['data']:
    if item['citingPaper']['year']:
        nums[item['citingPaper']['year']] = nums.setdefault(item['citingPaper']['year'],0) + 1

nums = {k:v for k,v in nums.items() if k >= df.loc[idx,'year']}
x = np.arange(min(nums),2022)
y = [nums[l] if l in nums else 0 for l in x]

x = np.arange(1,2022-min(nums)+1)
y_cumsum = np.cumsum(y)

coes,cov = curve_fit(func,x,y_cumsum)
print('title:',df.loc[idx,'title'])
print("lambda,mu,sigma:",coes)

plot_longterm_citation(x,y_cumsum,func,coes)
# case: longterm citaion is much greater than current citation
idx = 12
nums = {}
for item in  df.loc[idx,'citation_per_year']['data']:
    if item['citingPaper']['year']:
        nums[item['citingPaper']['year']] = nums.setdefault(item['citingPaper']['year'],0) + 1

nums = {k:v for k,v in nums.items() if k >= df.loc[idx,'year']}
x = np.arange(min(nums),2022)
y = [nums[l] if l in nums else 0 for l in x]

x = np.arange(1,2022-min(nums)+1)
y_cumsum = np.cumsum(y)

coes,cov = curve_fit(func,x,y_cumsum)
print('title:',df.loc[idx,'title'])
print("lambda,mu,sigma:",coes)

plot_longterm_citation(x,y_cumsum,func,coes)
# in a few cases,scipy can not fit data well
# example as below, notice the warnings and cov matrix
# some other tool may help, see in longterm_citation_wl.ipynb

idx = 13
nums = {}
for item in  df.loc[idx,'citation_per_year']['data']:
    if item['citingPaper']['year']:
        nums[item['citingPaper']['year']] = nums.setdefault(item['citingPaper']['year'],0) + 1

nums = {k:v for k,v in nums.items() if k >= df.loc[idx,'year']}
x = np.arange(min(nums),2022)
y = [nums[l] if l in nums else 0 for l in x]

x = np.arange(1,2022-min(nums)+1)
y_cumsum = np.cumsum(y)

coes,cov = curve_fit(func,x,y_cumsum)
print('title:',df.loc[idx,'title'])
print("lambda,mu,sigma:",coes)
print('cov:',cov)

plot_longterm_citation(x,y_cumsum,func,coes)<jupyter_output>F:\Anaconda\envs\AI\lib\site-packages\scipy\optimize\minpack.py:828: OptimizeWarning: Covariance of the parameters could not be estimated
  warnings.warn('Covariance of the parameters could not be estimated',
{'fraction_non_alphanumeric': 0.18414143182215997, 'fraction_numerical': 0.02205496236653247, 'mean_word_length': 4.679920477137177, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '194025', 'license': 'Apache-2.0', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2234, 'n_words': 392, 'path': 'longterm_citation.ipynb', 'repo_name': 'socrateslab/longterm_citation', 'size': '5713'}