<commit_before>"""
>>> from pyspark.context import SparkContext
>>> from pyspark.sql import SQLContext, Row, DataFrame
>>> sc = SparkContext('local', 'test')
...
>>> sc.setLogLevel("ERROR")
>>> sqlCtx = SQLContext(sc)
...
>>> rdd = sc.parallelize(range(1, 100)).map(lambda x: Row(i = x))
>>> df = rdd.toDF()
>>> df2 = cutLineage(df)
>>> df.head() == df2.head()
True
>>> df.schema == df2.schema
True
"""

from pyspark.sql import DataFrame

def cutLineage(df):
    """
    Cut the lineage of a DataFrame - used for iterative algorithms
    
    .. Note: This uses internal members and may break between versions
    """
    jRDD = df._jdf.toJavaRDD()
    jSchema = df._jdf.schema()
    jRDD.cache()
    sqlCtx = df.sql_ctx
    javaSqlCtx = sqlCtx._jsqlContext
    newJavaDF = javaSqlCtx.createDataFrame(jRDD, jSchema)
    newDF = DataFrame(newJavaDF, sqlCtx)
    return newDF<commit_msg>Add tags<commit_after>"""
>>> from pyspark.context import SparkContext
>>> from pyspark.sql import SQLContext, Row, DataFrame
>>> sc = SparkContext('local', 'test')
...
>>> sc.setLogLevel("ERROR")
>>> sqlCtx = SQLContext(sc)
...
>>> rdd = sc.parallelize(range(1, 100)).map(lambda x: Row(i = x))
>>> df = rdd.toDF()
>>> df2 = cutLineage(df)
>>> df.head() == df2.head()
True
>>> df.schema == df2.schema
True
"""

from pyspark.sql import DataFrame

#tag::cutLineage[]
def cutLineage(df):
    """
    Cut the lineage of a DataFrame - used for iterative algorithms
    
    .. Note: This uses internal members and may break between versions
    """
    jRDD = df._jdf.toJavaRDD()
    jSchema = df._jdf.schema()
    jRDD.cache()
    sqlCtx = df.sql_ctx
    javaSqlCtx = sqlCtx._jsqlContext
    newJavaDF = javaSqlCtx.createDataFrame(jRDD, jSchema)
    newDF = DataFrame(newJavaDF, sqlCtx)
    return newDF
#end::cutLineage[]
{'fraction_non_alphanumeric': 0.158624514697726, 'fraction_numerical': 0.00776483638380477, 'mean_word_length': 3.722513089005236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1995290', 'n_tokens_mistral': 638, 'n_tokens_neox': 635, 'n_words': 154}