<commit_before>//---------------------------------------------------------- -*- Mode: C++ -*-
// $Id$ 
//
// Created 2006/03/22
// Author: <NAME>
//
// Copyright 2008 Quantcast Corp.
// Copyright 2006-2008 Kosmix Corp.
//
// This file is part of Kosmos File System (KFS).
//
// Licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// 
//----------------------------------------------------------------------------

extern "C" {
#include <signal.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <openssl/hmac.h>
#include <openssl/md5.h>
}

#include <string>
#include <vector>

#include "common/properties.h"
#include "libkfsIO/DiskManager.h"
#include "libkfsIO/NetManager.h"
#include "libkfsIO/Globals.h"

#include "ChunkServer.h"
#include "ChunkManager.h"

using namespace KFS;
using namespace KFS::libkfsio;
using std::string;
using std::vector;
using std::cout;
using std::endl;

string gLogDir;
vector<string> gChunkDirs;
string gMD5Sum;

ServerLocation gMetaServerLoc;
int64_t gTotalSpace;			// max. storage space to use
int gChunkServerClientPort;	// Port at which kfs clients connect to us
string gChunkServerHostname;	// Our hostname to use (instead of using gethostname() )

Properties gProp;
const char *gClusterKey;
int gChunkServerRackId;
int gChunkServerCleanupOnStart;

int ReadChunkServerProperties(char *fileName);
static void computeMD5(const char *pathname);

int
main(int argc, char **argv)
{
    if (argc < 2) {
        cout << "Usage: " << argv[0] << " <properties file> {<msg log file>}" << endl;
        exit(0);
    }

    if (argc > 2) {
        KFS::MsgLogger::Init(argv[2]);
    } else {
        KFS::MsgLogger::Init(NULL);
    }

    if (ReadChunkServerProperties(argv[1]) != 0) {
        cout << "Bad properties file: " << argv[1] << " aborting...\n";
        exit(-1);
    }
    // Initialize things...
    libkfsio::InitGlobals();
    
    // for writes, the client is sending WRITE_PREPARE with 64K bytes;
    // to enable the data to fit into a single buffer (and thereby get
    // the data to the underlying FS via a single aio_write()), allocate a bit
    // of extra space. 64K + 4k
    libkfsio::SetIOBufferSize(69632);

    globals().diskManager.InitForAIO();

    // would like to limit to 200MB outstanding
    // globals().netManager.SetBacklogLimit(200 * 1024 * 1024);

    // compute the MD5 of the binary
    computeMD5(argv[0]);

    cout << "md5sum that send to metaserver: " << gMD5Sum << endl;

    gChunkServer.Init();
    gChunkManager.Init(gChunkDirs, gTotalSpace);
    gLogger.Init(gLogDir);
    gMetaServerSM.SetMetaInfo(gMetaServerLoc, gClusterKey, gChunkServerRackId, gMD5Sum);

    signal(SIGPIPE, SIG_IGN);

    // gChunkServerCleanupOnStart is a debugging option---it provides
    // "silent" cleanup
    if (gChunkServerCleanupOnStart == 0) {
        gChunkManager.Restart();
    }

    gChunkServer.MainLoop(gChunkServerClientPort, gChunkServerHostname);

    return 0;
}

static void
computeMD5(const char *pathname)
{
    MD5_CTX ctx;
    struct stat s;
    int fd;
    unsigned char md5sum[MD5_DIGEST_LENGTH];

    if (stat(pathname, &s) != 0)
	return;
    
    fd = open(pathname, O_RDONLY);
    MD5_Init(&ctx);
    unsigned char *buf = (unsigned char *) mmap(0, s.st_size, PROT_EXEC, MAP_SHARED, fd, 0);
    if (buf != NULL) {
        MD5(buf, s.st_size, md5sum);
        munmap(buf, s.st_size);
        char md5digest[2 * MD5_DIGEST_LENGTH + 1];
        md5digest[2 * MD5_DIGEST_LENGTH] = '\0';
        for (uint32_t i = 0; i < MD5_DIGEST_LENGTH; i++)
            sprintf(md5digest + i * 2, "%02x", md5sum[i]);
        gMD5Sum = md5digest;
        cout << "md5sum calculated from binary: " << gMD5Sum << endl;
    }
    close(fd);
}

static bool
make_if_needed(const char *dirname, bool check)
{<commit_msg> -- Increase core dump size to unlimited; print out a message when the server starts up<commit_after>//---------------------------------------------------------- -*- Mode: C++ -*-
// $Id$ 
//
// Created 2006/03/22
// Author: <NAME>
//
// Copyright 2008 Quantcast Corp.
// Copyright 2006-2008 Kosmix Corp.
//
// This file is part of Kosmos File System (KFS).
//
// Licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// 
//----------------------------------------------------------------------------

extern "C" {
#include <signal.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <openssl/hmac.h>
#include <openssl/md5.h>
#include <sys/time.h>
#include <sys/resource.h>
}

#include <string>
#include <vector>

#include "common/properties.h"
#include "libkfsIO/DiskManager.h"
#include "libkfsIO/NetManager.h"
#include "libkfsIO/Globals.h"

#include "ChunkServer.h"
#include "ChunkManager.h"

using namespace KFS;
using namespace KFS::libkfsio;
using std::string;
using std::vector;
using std::cout;
using std::endl;

string gLogDir;
vector<string> gChunkDirs;
string gMD5Sum;

ServerLocation gMetaServerLoc;
int64_t gTotalSpace;			// max. storage space to use
int gChunkServerClientPort;	// Port at which kfs clients connect to us
string gChunkServerHostname;	// Our hostname to use (instead of using gethostname() )

Properties gProp;
const char *gClusterKey;
int gChunkServerRackId;
int gChunkServerCleanupOnStart;

int ReadChunkServerProperties(char *fileName);
static void computeMD5(const char *pathname);

int
main(int argc, char **argv)
{
    if (argc < 2) {
        cout << "Usage: " << argv[0] << " <properties file> {<msg log file>}" << endl;
        exit(0);
    }

    if (argc > 2) {
        KFS::MsgLogger::Init(argv[2]);
    } else {
        KFS::MsgLogger::Init(NULL);
    }

    // set the coredump size to unlimited
    struct rlimit rlim;
    int err;
    rlim.rlim_cur = RLIM_INFINITY;
    rlim.rlim_max = RLIM_INFINITY;
    err = setrlimit(RLIMIT_CORE, &rlim);
    if (err) {
        KFS_LOG_VA_INFO("Unable to increase coredump file size: %d", err);
    }

    if (ReadChunkServerProperties(argv[1]) != 0) {
        cout << "Bad properties file: " << argv[1] << " aborting...\n";
        exit(-1);
    }

    // Initialize things...
    libkfsio::InitGlobals();

    KFS_LOG_INFO("Starting chunkserver...");
    
    // for writes, the client is sending WRITE_PREPARE with 64K bytes;
    // to enable the data to fit into a single buffer (and thereby get
    // the data to the underlying FS via a single aio_write()), allocate a bit
    // of extra space. 64K + 4k
    libkfsio::SetIOBufferSize(69632);

    globals().diskManager.InitForAIO();

    // would like to limit to 200MB outstanding
    // globals().netManager.SetBacklogLimit(200 * 1024 * 1024);

    // compute the MD5 of the binary
    computeMD5(argv[0]);

    KFS_LOG_VA_INFO("md5sum to send to metaserver: %s", gMD5Sum.c_str());

    gChunkServer.Init();
    gChunkManager.Init(gChunkDirs, gTotalSpace);
    gLogger.Init(gLogDir);
    gMetaServerSM.SetMetaInfo(gMetaServerLoc, gClusterKey, gChunkServerRackId, gMD5Sum);

    signal(SIGPIPE, SIG_IGN);

    // gChunkServerCleanupOnStart is a debugging option---it provides
    // "silent" cleanup
    if (gChunkServerCleanupOnStart == 0) {
        gChunkManager.Restart();
    }

    gChunkServer.MainLoop(gChunkServerClientPort, gChunkServerHostname);

    return 0;
}

static void
computeMD5(const char *pathname)
{
    MD5_CTX ctx;
    struct stat s;
    int fd;
    unsigned char md5sum[MD5_DIGEST_LENGTH];

    if (stat(pathname, &s) != 0)
	return;
    
    fd = open(pathname, O_RDONLY);
    MD5_Init(&ctx);
    unsigned char *buf = (unsigned char *) mmap(0, s.st_size, PROT_EXEC, MAP_SHARED, fd, 0);
    if (buf != NULL) {
        MD5(buf, s.st_size, md5sum);
        munmap(buf, s.st_size);
        char md5digest[2 * MD5_DIGEST_LENGTH + 1];
        md5digest[2 * MD5_DIGEST_LENGTH] = '\0';
        for (uint32_t i = 0; i < MD5_DIGEST_LENGTH; i++)
            sprintf(md5digest + i * 2, "%02x", md5sum[i]);
        gMD5Sum = md5digest;
        KFS_LOG_VA_DEBUG("md5sum calculated from binary: %s", gMD5Sum.c_str());
    }
    close(fd);
}

static bool
make_if_needed(const char *dirname, bool check)
{
{'fraction_non_alphanumeric': 0.1537190082644628, 'fraction_numerical': 0.022258953168044077, 'mean_word_length': 2.870362473347548, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4018844', 'n_tokens_mistral': 3138, 'n_tokens_neox': 2971, 'n_words': 865}