<jupyter_start><jupyter_text>multiprocessing<jupyter_code>import time
from multiprocess import Pool
COUNT = 10000000
start = time.perf_counter()
def countdown(n):
    while n > 0:
        n -= 1

pool = Pool(processes=2)
start = time.perf_counter()
r1 = pool.apply_async(countdown, [COUNT//2])
r2 = pool.apply_async(countdown, [COUNT//2])
pool.close()
pool.join()
print('Time used:', time.perf_counter() - start)
from multiprocess import Process,Pool,Queue

def f(x):
    return x*x

if __name__ == '__main__':
    with Pool(5) as p:
        print(p.map(f, [1, 2, 3]))
def f(q):
    q.put('hello world')
    
def other_function():
    q = Queue()
    p = Process(target=f, args=[q])
    p.start()
    print (q.get())
    p.join()
    
if __name__ == '__main__':
    other_function()
import torch
import torch.nn as nn
import torch.multiprocessing as mp
#import multiprocess as mp

import lib.finance.train_extern_functions as extern
from lib.finance.train_extern_functions import*
"""
#def ex_forward(m:nn.Module,queue:mp.Queue):
def ex_forward(model,queue):
    #t = queue.get()*2
    #queue.put(t)
    #queue.put("hello world")
    queue.put(f"hello world:\n{model(queue.get())}")
    #queue.put( 2+queue.get() )
"""

def f(q):
   #q.put('hello world')
    
    q.put(f'hello world:\n{q.get()}')

class Model(nn.Module):
    def __init__(self,dim1,dim2):
        super().__init__()
        self.l = nn.Linear(dim1,dim2)
    def forward(self,x):
        return self.l(x)
    
def other_function():
    dim1,dim2=3,2
    x = torch.rand(dim2,dim1)
    print(f"initialized x :\n{x}\n")
    stems = [Model(dim1,dim2),Model(dim1,dim2)]
    processes = []
    queues =[]
    for s in stems:
        s.share_memory()
        q = mp.Queue()
        queues.append(q)
        q.put(x)
        p = mp.Process(target=extern.ex_forward, args=[s,q])
        #p = mp.Process(target=f, args=[q])
        p.start()
        processes.append(p)
    for p in processes:
        p.join()
        
    #x_stem = queues[0].get()
    for q in queues: #[1:]:
        if not q.empty():
            print(q.get())
        #x_stem = x_stem + q.get()
    #print(f"processed x :{x_stem}")
    
if __name__ == '__main__':
    #mp.set_start_method("fork")
    other_function()    
#dim1,dim2=3,2
#model = Model(3,2)
#x = torch.rand(dim2,dim1)
#model(x)
import torch.multiprocessing as mp
from model import MyModel

def train(model):
    # Construct data_loader, optimizer, etc.
    for data, labels in data_loader:
        optimizer.zero_grad()
        loss_fn(model(data), labels).backward()
        optimizer.step()  # This will update the shared parameters

if __name__ == '__main__':
    num_processes = 4
    model = MyModel()
    # NOTE: this is required for the ``fork`` method to work
    model.share_memory()
    processes = []
    for rank in range(num_processes):
        p = mp.Process(target=train, args=(model,))
        p.start()
        processes.append(p)
    for p in processes:
        p.join()
import time
import numpy as np
n=int(1e6)
print(n)
%time np.random.uniform(low=0, high=1, size=n)
r_indices = np.random.randint(0,n, n)
r_indices
import torch
import torch.nn as nn
import torch.nn.functional as F
data = torch.tensor([[1.,2],[3.,4]])
data = data.unsqueeze(dim=1)#.unsqueeze(dim=0)
print(data.shape)

seq_length=2
embedding_width=30
nb_stems = 4
m = nn.Sequential(
     nn.Linear(seq_length,embedding_width*nb_stems)
     #
)
#m = nn.AvgPool1d(3, stride=3)
data=m(data)
print(f"after nn.linear: {data.shape}")
m = nn.AvgPool1d(kernel_size=nb_stems)
data = m(data)
print(f"after nn.AvgPool1d: {data.shape}")
m = nn.BatchNorm1d(1)
data = m(data)
print(f"after nn.BatchNorm1d {data.shape}")
from dataclasses import dataclass, field

@dataclass
class ValueRange:
    default:int=0
    range_:range = range(30, 180, 30)
        
@dataclass
class TrainingParams:
    seq_length:ValueRange = ValueRange(0,range(2,4,2))
    training_columns: list = field(default_factory=list)
tp = TrainingParams()
print(f"{tp}")
    
#ValueRange(0,range(2,4,2))
tp.seq_length.default
import random
%timeit 1 if random.random() < 0.5 else -1
%timeit [-1,1][random.randrange(2)]<jupyter_output>158 ns ± 17.5 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
876 ns ± 48.3 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
{'fraction_non_alphanumeric': 0.15103080843178132, 'fraction_numerical': 0.028028723650683346, 'mean_word_length': 3.3616161616161615, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '292535', 'license': 'Apache-2.0', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1632, 'n_words': 375, 'path': 'parallel.ipynb', 'repo_name': 'kasparlund/ai', 'size': '4317'}