<jupyter_start><jupyter_text><jupyter_code>'''
ABOUT THE MODULE:
This module contains logic to analyze about 50000 article texts
from "CyberLeninka" as a Russian scientific database
'''

'''
NOTATIONS:
– df – a dataframe/dataset, or an array of initial data.
'''
from bs4 import BeautifulSoup
import urllib.request
import requests
import csv
import pandas as pd
import random
import time
import os
import numpy as np
import statistics

# !pip uninstall pandas_profiling
# !pip install pandas_profiling
from pandas_profiling import ProfileReport

# !pip uninstall nltk
# !pip install nltk
import nltk
# nltk.download('punkt')
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist
'''
Initialize the dataframe
'''
df = pd.read_csv(
  '/content/drive/MyDrive/Science/Datasets/cyberleninka-sociology-articles/cyberleninka-sociology-articles'
  + '__2-0_articles-content/integrated-dataframes/integrated-dataframe__actuality-09-01-2021-2.csv'
)
'''
Open the dataframe
'''
df
'''
Create decorder for texts from string to array type
'''
firstArticleText = df['article_text'][0]

def transformFromStrToArr(textAsStr):
  textWithoutOuterWrapper = textAsStr.replace(']', '').replace('[', '')
  textAsArr = textWithoutOuterWrapper.replace("'", '').split(', ')
  return textAsArr

transformFromStrToArr(firstArticleText)
'''
Open first article text
'''
firstArticleText
'''
Restore text structure of an one random text
'''
textAsArr = transformFromStrToArr(firstArticleText)

SPACE = ' '
HYPHEN_1 = '-'
HYPHEN_2 = '–'
HYPHEN_3 = '—'

'''
Additional functions
'''
def isLastTextChunk(idx, textAsArr):
  idx == len(textAsArr) - 1

def isEndsOnHyphen(textAsStr):
  lastChunkSymbol = textAsArr[len(textAsArr) - 1]
  if (
    lastChunkSymbol == HYPHEN_1
    or lastChunkSymbol == HYPHEN_2
    or lastChunkSymbol == HYPHEN_3
  ):
    return true

'''
Main function
'''
def restoreOneArticleText(textAsArr):
  restoredText = ''

  for idx, textChunk in enumerate(textAsArr):
    
    # Pass last text chunk because it doesn't require handling for chunk end
    if isLastTextChunk(idx, textChunk):
      restoredText += textChunk
    
    if isEndsOnHyphen(textChunk):
      textChunkWithoutHyphen = textChunk[:-1]
      restoredText += textChunkWithoutHyphen

    # Handle other universal cases
    else:
      restoredText += textChunk + SPACE

  return restoredText

restoreOneArticleText(textAsArr)
'''
Integrate all articles texts in one text
'''
all_article_texts = df['article_text']
integrated_text = '' # This variable integrates all 46957 articles texts in same one
SPACE = ' '
file_to_write = open(
  '/content/drive/MyDrive/Science/Datasets/cyberleninka-sociology-articles/cyberleninka-sociology-articles'
  + '__2-0_articles-content/integrated-dataframes/all-article-texts-in-one-text__actuality-25-02-2021.txt',
  'a'
)

for article_text in all_article_texts:
  try:
    textAsArr = transformFromStrToArr(article_text)
    restoredText = restoreOneArticleText(textAsArr)
    file_to_write.write(restoredText + SPACE)
    file_to_write.close()
  except:
    pass

integrated_text<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1017274472168906, 'fraction_numerical': 0.013435700575815739, 'mean_word_length': 4.9789674952198855, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '168736', 'license': 'CC-BY-3.0', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1025, 'n_words': 249, 'path': 'analyze_articles_text.ipynb', 'repo_name': 'R-Mosolov/sociology-scientometric-analysis', 'size': '3126'}