|
"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[483],{89733:function(e,r,n){n.d(r,{z:function(){return a}});var t=n(57437),o=n(2265),s=n(71538),i=n(12218),l=n(49354);let d=(0,i.j)("inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50",{variants:{variant:{default:"bg-primary text-primary-foreground shadow hover:bg-primary/90",destructive:"bg-destructive text-destructive-foreground shadow-sm hover:bg-destructive/90",outline:"border border-input bg-background shadow-sm hover:bg-accent hover:text-accent-foreground",secondary:"bg-secondary text-secondary-foreground shadow-sm hover:bg-secondary/80",ghost:"hover:bg-accent hover:text-accent-foreground",link:"text-primary underline-offset-4 hover:underline"},size:{default:"h-9 px-4 py-2",sm:"h-8 rounded-md px-3 text-xs",lg:"h-10 rounded-md px-8",icon:"h-9 w-9"}},defaultVariants:{variant:"default",size:"default"}}),a=o.forwardRef((e,r)=>{let{className:n,variant:o,size:i,asChild:a=!1,...g}=e,m=a?s.g7:"button";return(0,t.jsx)(m,{className:(0,l.cn)(d({variant:o,size:i,className:n})),ref:r,...g})});a.displayName="Button"},4919:function(e,r,n){n.d(r,{g:function(){return i}});var t=n(57437),o=n(2265),s=n(49354);let i=o.forwardRef((e,r)=>{let{className:n,...o}=e;return(0,t.jsx)("textarea",{className:(0,s.cn)("flex min-h-[60px] w-full rounded-md border border-input bg-transparent px-3 py-2 text-sm shadow-sm placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50",n),ref:r,...o})});i.displayName="Textarea"},80018:function(e,r,n){n.d(r,{X:function(){return d}});var t=n(28580),o=n(36393),s=n(13201),i=n(62052);let l=(0,i.hu)("Embedding Engine",i.$5.embeddingEngine.main);class d extends o.Z{getEmbeddingEngineLogs(e){return this.embeddingEngineLog.slice(-e)}getAvailableModels(){return Array.from(new Set(Object.keys(this.embeddingWorkers).map(e=>this.embeddingWorkers[e].modelName)))}logEngineEvent(e){e.at||(e.at=new Date);let r=this.embeddingEngineLog.length;return l.debug("Embedding engine event ".concat(e.type)),this.embeddingEngineLog.push(e),r}async scaleEmbeddingWorkers(e,r){let n=Object.values(this.embeddingWorkers).filter(r=>r.modelName===e).length;if(n!==r){if(n<r){l.debug("Scaling up number of embedding workers for ".concat(e," to ").concat(r));for(let t=0;t<r-n;t++){let r="embedding-".concat(e,"-").concat((0,s.zs)());this.addEmbeddingWorker(e,r)}}else for(let t of(l.debug("Scaling down number of embedding workers for ".concat(e," to ").concat(r)),Object.keys(this.embeddingWorkers).sort((e,r)=>this.embeddingWorkers[e].busy===this.embeddingWorkers[r].busy?0:this.embeddingWorkers[e].busy?-1:1).slice(0,n-r)))this.deleteEmbeddingWorker(t)}}addEmbeddingWorker(e,r){if(this.embeddingWorkers[r])return;l.debug("Trying to create new embedding worker ".concat(r," for model ").concat(e));let o=new Worker(n.tu(new URL(n.p+n.u(99),n.b)));o.onmessage=n=>{let t=n.data;switch(l.debug("Received message from worker ".concat(r,": ").concat(t.type)),t.type){case"workerLoaded":this.embeddingWorkers[r].status="loaded",this.embeddingWorkers[r].busy=!1,this.logEngineEvent({type:"embeddings_worker_loaded",modelName:e,workerId:r}),this.embeddingWorkers[r].workerLoadedPromise.resolve(!0),this.emit("workerFree",{modelName:e,workerId:r});break;case"workerLoadFailure":this.embeddingWorkers[r].status="failed",this.logEngineEvent({type:"engine_loading_error",modelName:e,error:"Failed to load embedding worker",workerId:r}),this.embeddingWorkers[r].workerLoadedPromise.resolve(!1),this.logEngineEvent({type:"embeddings_worker_unload",workerId:r}),delete this.embeddingWorkers[r];break;case"embeddingSuccess":let o=this.embeddingJobQueue.find(e=>e.batchId===t.batchId);o?o.completionPromise.resolve(t.results):l.error("EMBEDDING ENGINE ERROR: SHOUDLNT HAPPEN, couldn't find job to resolve"),this.emit("workerFree",{modelName:e,workerId:r});break;case"embeddingFailure":let s=this.embeddingJobQueue.find(e=>e.assignedWorkerId===r);s&&s.completionPromise.resolve(!1),this.emit("workerFree",{modelName:e,workerId:r});break;case"workerBusyEmbedding":this.embeddingWorkers[r].busy=!0;break;case"workerIdle":this.embeddingWorkers[r].busy=!1,this.runJobFromQueue()}},o.postMessage({type:"loadWorker",modelName:e,workerId:r}),this.embeddingWorkers[r]={busy:!0,worker:o,modelName:e,status:"loading",workerLoadedPromise:new t.C}}deleteEmbeddingWorker(e){this.embeddingWorkers[e]&&(this.embeddingWorkers[e].worker.terminate(),delete this.embeddingWorkers[e],this.logEngineEvent({type:"embeddings_worker_unload",workerId:e}))}async embedText(e,r){let n=new t.C;return this.embeddingJobQueue.push({batchId:"".concat(this.embeddingBatchCounter++),modelName:r,params:{texts:e},completionPromise:n}),this.queuesRunning<Object.keys(this.embeddingWorkers).length&&this.runJobFromQueue(),n.promise}async runJobFromQueue(){try{this.queuesRunning++,l.debug("Trying to run a job from the queue, queue length is ".concat(this.embeddingJobQueue.length," jobs, with ").concat(this.queuesRunning," queues running"));let e=this.embeddingJobQueue.filter(e=>!e.assignedWorkerId);if(0===e.length){l.debug("No jobs left, queue is going to sleep"),this.queuesRunning--;return}let r=e.shift(),n=Object.keys(this.embeddingWorkers).filter(e=>this.embeddingWorkers[e].modelName===r.modelName);if(0===n.length)l.error("No workers loaded with embedding model ".concat(r.modelName,", ignoring job")),r.completionPromise.resolve(!1);else{l.debug("".concat(n.length," workers available for embedding ").concat(r.params.texts));let t=n.filter(e=>this.embeddingWorkers[e]&&this.embeddingWorkers[e].modelName===r.modelName&&"loaded"===this.embeddingWorkers[e].status&&!this.embeddingWorkers[e].busy);if(0===t.length){e.unshift(r),l.debug("No free workers available, wait to be called on idle"),this.queuesRunning--;return}let o=t[Math.floor(Math.random()*t.length)];r.assignedWorkerId=o,this.embeddingJobQueue.push(r),r.params.texts.forEach((e,n)=>{this.logEngineEvent({type:"engine_embedding_start",text:e,batchId:r.batchId,workerId:o})}),l.debug("Embedding ".concat(r.batchId,": ").concat(r.params.texts.length," texts with ").concat(r.modelName," on worker ").concat(o));try{this.embeddingWorkers[o].worker.postMessage({type:"embedText",texts:r.params.texts,batchId:r.batchId});let e=await r.completionPromise.promise;return e?e.forEach((e,n)=>{this.logEngineEvent({type:"engine_embedding_success",bEmbeddingHash:e.bEmbeddingHash,batchId:r.batchId,workerId:o})}):r.params.texts.forEach((e,n)=>{this.logEngineEvent({type:"engine_embedding_error",error:"Failed to embed text, returned false",batchId:r.batchId,workerId:o})}),this.queuesRunning--,e}catch(e){r.params.texts.forEach((n,t)=>{this.logEngineEvent({type:"engine_embedding_error",error:e,batchId:r.batchId,workerId:o})}),r.completionPromise.resolve(!1),this.queuesRunning--}}}catch(e){l.error("Error running job from queue",e),this.logEngineEvent({type:"engine_embedding_error",error:e,batchId:"unassigned",workerId:"unassigned"}),this.queuesRunning--}}constructor(...e){super(...e),this.embeddingEngineLog=[],this.embeddingBatchCounter=0,this.queuesRunning=0,this.embeddingWorkers={},this.embeddingJobQueue=[]}}},89183:function(e,r,n){n.d(r,{g:function(){return u}});var t=n(28580),o=n(31561),s=n(36393),i=n(13201),l=n(62052),d=n(89028),a=n(22170),g=n.n(a);let m=(0,l.hu)("LLM Engine",l.$5.llmEngine.main),c=(0,d.ri)().llmEngineSettings;class u extends s.Z{logEngineEvent(e){e.at||(e.at=new Date);let r=this.engineLog.length;return m.debug("Engine event ".concat(e.type),e),this.engineLog.push(e),this.engineLog.length>c.engineLogLimit&&(this.engineLog=this.engineLog.slice(-c.engineLogLimit)),r}parseCustomLoadingProgress(e){let r=/Loading model from cache\[(\d+)\/(\d+)\]/.exec(e);if(r)try{let e=parseInt(r[1]),n=parseInt(r[2]);return e/n-.01}catch(e){return}}searchEngineLogs(e,r){return this.engineLog.filter(n=>n.type===e&&n.workerId===r)}getEngineLogs(e){return this.engineLog.slice(-e)}updateStreamingLogResult(e,r){let n=this.engineLog[r];"engine_inference_streaming_result"===n.type&&("token"===e.type?(n.result+=e.token,n.tokenCount++):"fullMessage"===e.type?(n.result=e.message,n.completed=!0):"tokenCount"===e.type&&(n.tokenCount=e.tokenCount))}getWorkerStates(){return Object.keys(this.llmWorkers).reduce((e,r)=>(e[r]={modelName:this.llmWorkers[r].modelName,state:this.llmWorkers[r].inferenceInProgress?"inference-in-progress":this.llmWorkers[r].modelLoadingProgress<1?"loading":"idle",loadingProgress:this.llmWorkers[r].modelLoadingProgress},e),{})}getWorkerAvailability(e){return Object.values(this.llmWorkers).reduce((r,n)=>{if(e.includes(n.modelName)){var t,o;null!==(o=r[t=n.modelName])&&void 0!==o||(r[t]={count:0,free:0}),r[n.modelName].count++,!n.inferenceInProgress&&n.modelLoadingProgress>=1&&r[n.modelName].free++}return r},{})}async unloadWorker(e){let r=arguments.length>1&&void 0!==arguments[1]&&arguments[1];if(this.llmWorkers[e]){var n,t;r||await (null===(n=this.llmWorkers[e].inferencePromise)||void 0===n?void 0:n.promise),null===(t=this.llmWorkers[e].llmEngine)||void 0===t||t.unload(),delete this.llmWorkers[e],this.logEngineEvent({type:"engine_unload",workerId:e}),this.emit("workerUnloaded",{workerId:e}),this.saveWorkersToSettings()}}saveWorkersToSettings(){let e=Object.values(Object.values(this.llmWorkers).filter(e=>e.modelLoadingProgress>=1).reduce((e,r)=>(e[r.modelName]?e[r.modelName].count++:e[r.modelName]={modelName:r.modelName,count:1},e),{}));(0,d.zQ)({workerSettings:{initialLLMWorkers:e}})}async getWorkerState(e){return this.llmWorkers[e]?this.llmWorkers[e].modelLoadingProgress<1?{state:"loading",loadingProgress:this.llmWorkers[e].modelLoadingProgress}:this.llmWorkers[e].inferenceInProgress?{state:"inference-in-progress"}:{state:"idle"}:null}async loadWorker(e,r){var s,i,l;if(null!==(i=(s=this.llmWorkers)[r])&&void 0!==i||(s[r]={modelName:e,modelLoadingProgress:0}),this.llmWorkers[r].modelLoadingPromise)return m.debug("Tried to create worker ".concat(r,", but creation is already done or in progress")),await this.llmWorkers[r].modelLoadingPromise.promise;this.llmWorkers[r].modelLoadingPromise=new t.C;try{this.logEngineEvent({type:"engine_loading",modelName:e,workerId:r}),this.emit("workerLoading",{modelName:e,workerId:r}),this.llmWorkers[r].llmEngine=await o.F0(new Worker(n.tu(new URL(n.p+n.u(661),n.b)),{type:void 0}),e,{initProgressCallback:n=>{if(m.debug("Worker ".concat(r,": Loading ").concat(e," progress - "),n),this.emitModelLoadingProgress(),0===n.progress){let e=this.parseCustomLoadingProgress(n.text);e?(this.llmWorkers[r].modelLoadingProgress=e,m.debug("Worker ".concat(r,": Custom progress parsed - ").concat(e))):this.llmWorkers[r].modelLoadingProgress=n.progress}else this.llmWorkers[r].modelLoadingProgress=n.progress;if(1===n.progress&&!this.searchEngineLogs("engine_loaded",r).filter(r=>r.modelName===e).length){var t;this.logEngineEvent({type:"engine_loaded",modelName:e,workerId:r}),this.emit("workerLoaded",{modelName:e,workerId:r}),this.emit("workerFree",{workerId:r}),this.saveWorkersToSettings(),null===(t=this.llmWorkers[r].modelLoadingPromise)||void 0===t||t.resolve(r)}}})}catch(n){this.emit("workerLoadFailed",{modelName:e,workerId:r,error:n}),this.logEngineEvent({type:"engine_loading_error",modelName:e,workerId:r,error:n}),m.error("Worker ".concat(r,": Error loading ").concat(e,": ").concat(n),n),null===(l=this.llmWorkers[r].modelLoadingPromise)||void 0===l||l.reject(n),this.unloadWorker(r,!0)}return this.llmWorkers[r]&&await this.llmWorkers[r].modelLoadingPromise.promise||!1}getMatchingWorkers(e,r){return Object.keys(this.llmWorkers).filter(n=>this.llmWorkers[n].modelName===e.modelName&&this.llmWorkers[n].modelLoadingProgress>=1&&(!r||!this.llmWorkers[n].inferenceInProgress)).reduce((e,r)=>(e[r]=this.llmWorkers[r],e),{})}async runInferenceNonStreaming(e){let r=await this.runInference(e),n="",t=0;for await(let e of r)if("fullMessage"===e.type)n=e.message;else{if("error"===e.type)return{success:!1,error:e.error};"tokenCount"===e.type?t=e.tokenCount:"token"===e.type&&(n+=e.token,t++)}return{success:!0,result:n,tokenCount:t}}async *runInference(e,r){let n=this.getMatchingWorkers(e,!0);if(0===Object.keys(n).length)throw Error("No free workers available");let t=Object.keys(n)[Math.floor(Math.random()*Object.keys(n).length)];for await(let n of(await this.runInferenceOnWorker(e,t,r)))yield n}abortWorkerInference(e){let r=this.llmWorkers[e];if(r&&r.llmEngine&&r.inferenceInProgress){var n;r.llmEngine.interruptGenerate(),r.inferenceInProgress=!1,null===(n=r.inferencePromise)||void 0===n||n.resolve(!1),this.logEngineEvent({type:"engine_inference_error",workerId:e,inferenceId:this.inferenceCounter,error:"Inference manually aborted with control signal from the engine"})}}async *runInferenceOnWorker(e,r,n){if(!this.llmWorkers[r]||!this.llmWorkers[r].llmEngine||!this.llmWorkers[r].modelLoadingPromise)throw Error("Engine not loaded");if(await this.llmWorkers[r].modelLoadingPromise.promise,this.llmWorkers[r].inferenceInProgress)throw Error("Inference already in progress");let o=this.inferenceCounter++;this.logEngineEvent({type:"engine_inference_start",workerId:r,inferenceId:o,params:e}),this.emit("workerBusy",{workerId:r}),this.llmWorkers[r].inferenceInProgress=!0,this.llmWorkers[r].inferencePromise=new t.C;let s=this.logEngineEvent({type:"engine_inference_streaming_result",workerId:r,inferenceId:o,completed:!1,tokenCount:0,result:""});try{let t=await this.llmWorkers[r].llmEngine.chat.completions.create({stream:!0,messages:e.messages,temperature:1,max_gen_len:2048}),i="",l=0;for await(let e of t){if(null==n?void 0:n.aborted){this.llmWorkers[r].llmEngine.interruptGenerate(),this.llmWorkers[r].inferenceInProgress=!1,this.llmWorkers[r].inferencePromise.resolve(!1),this.logEngineEvent({type:"engine_inference_error",workerId:r,inferenceId:o,error:"Inference manually aborted with control signal during inference"}),yield{type:"error",error:"Inference manually aborted with control signal"};return}if(e.choices[0].delta.content){let r={type:"token",token:e.choices[0].delta.content};i+=e.choices[0].delta.content,l++,this.updateStreamingLogResult(r,s),yield r}}let d={type:"tokenCount",tokenCount:l};this.updateStreamingLogResult(d,s),yield d;let a={type:"fullMessage",message:i};this.updateStreamingLogResult(a,s),yield a,this.llmWorkers[r].inferenceInProgress=!1,this.llmWorkers[r].inferencePromise.resolve(!0)}catch(e){m.error("Worker ".concat(r,": Error running inference"),e),this.llmWorkers[r].inferenceInProgress=!1,this.llmWorkers[r].inferencePromise.resolve(!1),this.logEngineEvent({type:"engine_inference_error",workerId:r,inferenceId:o,error:e}),yield{type:"error",error:e}}this.emit("workerFree",{workerId:r})}async scaleLLMWorkers(e,r){let n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];try{let t=Object.values(this.llmWorkers).filter(r=>r.modelName===e).length;if(t===r)return;if(t<r){m.debug("Scaling up number of llm workers for ".concat(e," to ").concat(r));let n=[];for(let o=0;o<r-t;o++){let r="llm-".concat(e,"-").concat((0,i.zs)());n.push(this.loadWorker(e,r))}}else{m.debug("Scaling down number of llm workers for ".concat(e," to ").concat(r));let o=Object.keys(this.llmWorkers).sort((e,r)=>this.llmWorkers[e].inferenceInProgress===this.llmWorkers[r].inferenceInProgress?0:this.llmWorkers[e].inferenceInProgress?-1:1).slice(0,t-r),s=[];for(let e of o)s.push(this.unloadWorker(e,n))}}catch(e){m.error("Error updating LLM workers",e)}}constructor(...e){super(...e),this.llmWorkers={},this.engineLog=[],this.inferenceCounter=0,this.emitModelLoadingProgress=g()(()=>{this.emit("modelLoadingProgress")},c.debounceLoadingProgressEventMs)}}},79798:function(e,r,n){n.d(r,{o:function(){return t}});let t=["Llama-3-8B-Instruct-q4f32_1","Llama-2-7b-chat-hf-q4f16_1","Llama-2-13b-chat-hf-q4f16_1","Mistral-7B-Instruct-v0.2-q4f16_1","Hermes-2-Pro-Mistral-7B-q4f16_1","gemma-2b-it-q4f16_1","TinyLlama-1.1B-Chat-v0.4-q0f16"]}}]); |
