<commit_before>/*
 *
 * CODENVY CONFIDENTIAL
 * ________________
 *
 * [2012] - [2015] Codenvy, S.A.
 * All Rights Reserved.
 * NOTICE: All information contained herein is, and remains
 * the property of Codenvy S.A. and its suppliers,
 * if any. The intellectual and technical concepts contained
 * herein are proprietary to Codenvy S.A.
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from Codenvy S.A..
 */

---------------------------------------------------------------------------
-- Loads resources.
-- @return {dt : datetime,  event : bytearray, message : chararray, user : bytearray, ws : bytearray}
-- In details:
--   field 'date' contains date in format 'YYYYMMDD'
--   field 'time' contains seconds from midnight
---------------------------------------------------------------------------
DEFINE loadResources(resourceParam, from, to, userType, wsType) RETURNS Y {
  l1 = LOAD '$resourceParam' USING PigStorage() as (message : chararray);
  l2 = FILTER l1 BY INDEXOF(message, 'EVENT#', 0) > 0;

  l3 = extractUser(l2, '$userType');
  l4 = extractWs(l3, '$wsType');
  l5 = extractParam(l4, 'WS-ID', 'wsId');
  l6 = extractParam(l5, 'USER-ID', 'userId');
  l7 = extractParam(l6, 'TIME', 'time');
  l = FOREACH l7 GENERATE user,
                          userId,
                          ws,
                          wsId,
                          message,
                          time,
                          REGEX_EXTRACT_ALL(message, '([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}) ([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3}).*\\s-(\\s.*)') AS pattern;

  k1 = FOREACH l GENERATE ReplaceUserWithId(user, userId) AS user,
                          ReplaceWsWithId(ws, wsId) AS ws,
                          (time IS NOT NULL ? ToDate((long)time) : ToDate(pattern.$1, 'yyyy-MM-dd HH:mm:ss,SSS')) AS dt,
                          pattern.$2 AS message;
  k2 = filterByDate(k1, '$from', '$to');
  k3 = extractParam(k2, 'EVENT', 'event');
  k = removeEmptyField(k3, 'event');

  m1 = FILTER k BY '$wsType' == 'ANY' OR  ws == 'default' OR
                    ('$wsType' == 'TEMPORARY' AND IsTemporaryWorkspaceById(ws)) OR
                    ('$wsType' == 'PERSISTENT' AND NOT IsTemporaryWorkspaceById(ws));

  m2 = FILTER m1 BY '$userType' == 'ANY' OR user == 'default' OR
                      ('$userType' == 'ANONYMOUS' AND IsAnonymousUserById(user)) OR
                      ('$userType' == 'REGISTERED' AND NOT IsAnonymousUserById(user));

  m = DISTINCT m2;
  $Y = FOREACH m GENERATE dt,
                          user,
                          event,
                          message,
                          ws;
};
---------------------------------------------------------------------------
-- Removes tuples with empty fields
---------------------------------------------------------------------------<commit_msg>Fix NPE in scripts, dt cann't be NULL<commit_after>/*
 *
 * CODENVY CONFIDENTIAL
 * ________________
 *
 * [2012] - [2015] Codenvy, S.A.
 * All Rights Reserved.
 * NOTICE: All information contained herein is, and remains
 * the property of Codenvy S.A. and its suppliers,
 * if any. The intellectual and technical concepts contained
 * herein are proprietary to Codenvy S.A.
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from Codenvy S.A..
 */

---------------------------------------------------------------------------
-- Loads resources.
-- @return {dt : datetime,  event : bytearray, message : chararray, user : bytearray, ws : bytearray}
-- In details:
--   field 'date' contains date in format 'YYYYMMDD'
--   field 'time' contains seconds from midnight
---------------------------------------------------------------------------
DEFINE loadResources(resourceParam, from, to, userType, wsType) RETURNS Y {
  l1 = LOAD '$resourceParam' USING PigStorage() as (message : chararray);
  l2 = FILTER l1 BY INDEXOF(message, 'EVENT#', 0) > 0;

  l3 = extractUser(l2, '$userType');
  l4 = extractWs(l3, '$wsType');
  l5 = extractParam(l4, 'WS-ID', 'wsId');
  l6 = extractParam(l5, 'USER-ID', 'userId');
  l7 = extractParam(l6, 'TIME', 'time');
  l = FOREACH l7 GENERATE user,
                          userId,
                          ws,
                          wsId,
                          message,
                          time,
                          REGEX_EXTRACT_ALL(message, '([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}) ([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3}).*\\s-(\\s.*)') AS pattern;

  k1 = FOREACH l GENERATE ReplaceUserWithId(user, userId) AS user,
                          ReplaceWsWithId(ws, wsId) AS ws,
                          (time IS NOT NULL ? ToDate((long)time) : ToDate(pattern.$1, 'yyyy-MM-dd HH:mm:ss,SSS')) AS dt,
                          pattern.$2 AS message;
  k2 = extractParam(k1, 'EVENT', 'event');
  k3 = removeEmptyField(k2, 'event');
  k4 = FILTER k3 BY dt IS NOT NULL;
  k = filterByDate(k4, '$from', '$to');


  m1 = FILTER k BY '$wsType' == 'ANY' OR  ws == 'default' OR
                    ('$wsType' == 'TEMPORARY' AND IsTemporaryWorkspaceById(ws)) OR
                    ('$wsType' == 'PERSISTENT' AND NOT IsTemporaryWorkspaceById(ws));

  m2 = FILTER m1 BY '$userType' == 'ANY' OR user == 'default' OR
                      ('$userType' == 'ANONYMOUS' AND IsAnonymousUserById(user)) OR
                      ('$userType' == 'REGISTERED' AND NOT IsAnonymousUserById(user));

  m = DISTINCT m2;
  $Y = FOREACH m GENERATE dt,
                          user,
                          event,
                          message,
                          ws;
};
---------------------------------------------------------------------------
-- Removes tuples with empty fields
---------------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.22317870237525905, 'fraction_numerical': 0.02359317710824167, 'mean_word_length': 1.6517328825021134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1058571', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1825, 'n_words': 543}