Upload folder using huggingface_hub

static/javascript/rag_cleaner.js

@@ -2,90 +2,115 @@
 
 const removeChars = (txt) => {
   const charsRm = /[\u00AD\u200B\u200C\u200D\u2060\uFEFF]/g;
-  txt = txt.replace(charsRm, '');
-  txt = Array.from(txt).filter(ch => !/^C/.test(ch.codePointAt(0))).join('');
+  txt = txt.replace(charsRm, "");
+  txt = Array.from(txt)
+    .filter((ch) => !/^C/.test(ch.codePointAt(0)))
+    .join("");
   return txt;
 };
 
 const replaceChars = (txt) => {
   const charsSrp = /[\u00A0\u2000-\u200A\u202F\u205F\u3000\t\r\f\v]/g;
-  txt = txt.replace(charsSrp, ' ');
-  txt = txt.replace(/ +/g, ' ');
+  txt = txt.replace(charsSrp, " ");
+  txt = txt.replace(/ +/g, " ");
   return txt.trim();
 };
 
 const removeTag = (txt) => {
-  txt = txt.replace(/<<<|>>>|<<|>>/g, '');
+  txt = txt.replace(/<<<|>>>|<<|>>/g, "");
   return txt;
 };
 
 function cleanDoc(txt) {
   try {
-      txt = removeTag(txt);
-      txt = removeChars(txt);
-      txt = replaceChars(txt);
-      txt = txt.replace(/\n/g, " ");
-      txt = txt.replace(/\t/g, " ");
-      txt = txt.replace(/ +/g, ' ');
-      txt = txt.replace(/\n\s*\n/g, '\n');
-      txt = txt.replace(/[“”]/g, '"');
-      txt = txt.replace(/[‘’]/g, "'");
-      txt = txt.replace(/[«»„“]/g, '"');
-      txt = txt.replace(/[–—]/g, '-');
-      txt = txt.replace(/\\[nrt]/g, '');
-      txt = txt.replace(/[^\w\sàèéìòùÀÈÉÌÒÙáéíóúÁÉÍÓÚäëïöüÄËÏÖÜâêîôûÂÊÎÔÛçÇñÑ.,;:!?'"()-]/g, '');
-      return txt.trim();
+    txt = removeTag(txt);
+    txt = removeChars(txt);
+    txt = replaceChars(txt);
+    txt = txt.replace(/\n/g, " ");
+    txt = txt.replace(/\t/g, " ");
+    txt = txt.replace(/ +/g, " ");
+    txt = txt.replace(/\n\s*\n/g, "\n");
+    txt = txt.replace(/[“”]/g, '"');
+    txt = txt.replace(/[‘’]/g, "'");
+    txt = txt.replace(/[«»„“]/g, '"');
+    txt = txt.replace(/[–—]/g, "-");
+    txt = txt.replace(/\\[nrt]/g, "");
+    txt = txt.replace(/[^\w\sàèéìòùÀÈÉÌÒÙáéíóúÁÉÍÓÚäëïöüÄËÏÖÜâêîôûÂÊÎÔÛçÇñÑ.,;:!?'"()-]/g, "");
+    return txt.trim();
   } catch (e) {
-      xerror(e);
-      return "Errore di codifica nel documento";
+    xerror(e);
+    return "Errore di codifica nel documento";
   }
-};
+}
 
-function cleanResponse(txt){
+function cleanResponse(txt) {
   try {
-      txt = removeChars(txt);
-      txt = replaceChars(txt);
-      txt = txt.replace(/\t/g, " ");
-      txt = txt.replace(/ +/g, ' ');
-      txt = txt.replace(/\n\s*\n/g, '\n');
-      txt = txt.replace(/[“”]/g, '"');
-      txt = txt.replace(/[‘’]/g, "'");
-      txt = txt.replace(/[«»„“]/g, '"');
-      txt = txt.replace(/[–—]/g, '-');
-      txt = txt.replace(/\\[nrt]/g, '');
-      txt = txt.replace(/[^\w\sàèéìòùÀÈÉÌÒÙáéíóúÁÉÍÓÚäëïöüÄËÏÖÜâêîôûÂÊÎÔÛçÇñÑ.,;:!?'"()-]/g, '');
-      return txt.trim();
+    txt = removeChars(txt);
+    txt = replaceChars(txt);
+    txt = txt.replace(/\t/g, " ");
+    txt = txt.replace(/ +/g, " ");
+    txt = txt.replace(/\n\s*\n/g, "\n");
+    txt = txt.replace(/[“”]/g, '"');
+    txt = txt.replace(/[‘’]/g, "'");
+    txt = txt.replace(/[«»„“]/g, '"');
+    txt = txt.replace(/[–—]/g, "-");
+    txt = txt.replace(/\\[nrt]/g, "");
+    txt = txt.replace(/[^\w\sàèéìòùÀÈÉÌÒÙáéíóúÁÉÍÓÚäëïöüÄËÏÖÜâêîôûÂÊÎÔÛçÇñÑ.,;:!?'"()-]/g, "");
+    return txt.trim();
   } catch (e) {
-      xerror(e);
-      return "Errore di codifica nela risposta";
+    xerror(e);
+    return "Errore di codifica nela risposta";
   }
-};
+}
 
-function list2text(lst) {
-  return lst.map(s => `${cleanResponse(s)}`).join('\n\n');
-};
+function cleanOutput(txt) {
+  try {
+    txt = removeChars(txt);
+    txt = replaceChars(txt);
+    txt = txt.replace(/\t/g, " ");
+    txt = txt.replace(/ +/g, " ");
+    txt = txt.replace(/\n\s*\n/g, "\n");
+    txt = txt.replace(/[“”]/g, '"');
+    txt = txt.replace(/[‘’]/g, "'");
+    txt = txt.replace(/[«»„“]/g, '"');
+    txt = txt.replace(/[–—]/g, "-");
+    txt = txt.replace(/\\[nrt]/g, "");
+    txt = txt.replace(/[^\w\sàèéìòùÀÈÉÌÒÙáéíóúÁÉÍÓÚäëïöüÄËÏÖÜâêîôûÂÊÎÔÛçÇñÑ.,;:!?'"()-]/g, "");
+    return txt.trim();
+  } catch (e) {
+    xerror(e);
+    return "Errore di codifica nell'output";
+  }
+}
+
+function list2text(docList) {
+  const docDict = {};
+
+  function getLabel(txt) {
+    const match = txt.match(/<<<(.*?)>>>/);
+    return match ? match[1] : null;
+  }
 
+  for (let doc of docList) {
+    const docName = getLabel(doc);
+    // const content = doc.replace(/<<<.*?>>>/g, ''); //TODO
+    const content = doc.replace(/<<<.*?>>>/, "");
+    if (docName in docDict) {
+      docDict[docName] += `\n ${content}`;
+    } else {
+      docDict[docName] = content;
+    }
+  }
+  const lst = [];
+  for (let [docName, content] of Object.entries(docDict)) {
+    lst.push(`Documento: ${docName}.\n${content}`);
+  }
+  return lst.join("\n\n");
+}
 
-function cleanOutput(text) {
-  // Rimuove gli spazi bianchi iniziali e finali
-  text = text.trim();
-  // text = removeLabels(text);// TODO
-  // Sostituisce le sequenze di più di due newline con due newline
-  text = text.replace(/\n2,}/g, '\n');
-  // text = text.replace(/\n{2,}/g, "\n");
-  // Rimuove gli spazi bianchi extra all'inizio di ogni riga
-  text = text.replace(/^[ \t]+/gm, "");
-  // Sostituisce i trattini bassi multipli con una linea orizzontale
-  // text = text.replace(/_{3,}/g, '\n\n' + '-'.repeat(40) + '\n\n');
-  // Formatta gli elenchi puntati per una migliore leggibilità
-  text = text.replace(/^(\s*[-*•])(\s*)/gm, "\n$1 ");
-  // Formatta gli elenchi numerati per una migliore leggibilità
-  text = text.replace(/^(\s*\d+\.)(\s*)/gm, "\n$1 ");
-  // Aggiunge una riga vuota prima e dopo i blocchi di codice
-  text = text.replace(/(```[\s\S]*?```)/g, "\n\n$1\n\n");
-  // Aggiunge un'andata a capo dopo ogni punto, eccetto quando seguito da newline o fine stringa
-  // text = text.replace(/\.(?!\n|$)/g, '.\n');
-  // Rimuove gli spazi bianchi extra alla fine di ogni riga
-  text = text.replace(/[ \t]+$/gm, "");
-  return text;
+// <<<doc_name>>> => Documento: doc_name
+function subResponseDOcTag(txt) {
+  const regex = /<<<(.*?)>>>/;
+  const result = txt.replace(regex, (match, p1) => `Documento: ${p1}`);
+  return result;
 }

static/javascript/rag_mgr.js

@@ -18,7 +18,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 "use strict";
-const MAX_PROMPT_LENGTH = 1024 * 80;
+const MAX_PROMPT_LENGTH = 1024 * 85;
 // decremento dopo errore per tokens eccessivi
 const PROMPT_DECR = 1024 * 2;
 
@@ -68,13 +68,13 @@ const Rag = {
     }, 0);
     return rspsl;
   },
-  log(msg, plft, prgt) {
+  ragLog(msg, lftLen, rgtLen) {
     const maxl = MAX_PROMPT_LENGTH;
     const rspsl = this.responsesLength();
     const d = "&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;";
-    let s = `${msg} mx:${maxl} lft:${plft} rgt:${prgt} arr:${rspsl}`;
+    let s = `${msg} mx:${maxl} lft:${lftLen} rgt:${rgtLen} arr:${rspsl}`;
     xlog(s);
-    s = `${msg}${d}${plft}${d}${prgt}${d}${rspsl}`;
+    s = `${msg}${d}${lftLen}${d}${rgtLen}${d}${rspsl}`;
     //nsg plft.length prgt.length responses.length
     UaLog.log(s);
   },
@@ -104,10 +104,10 @@ const Rag = {
     }
     return size;
   },
-  getPartDoc(doc, partSize) {
-    const part = doc.substring(0, partSize);
-    doc = doc.substring(partSize).trim();
-    return [part, doc];
+  getPartDoc(pRgt, partSize) {
+    const pLft = pRgt.substring(0, partSize);
+    pRgt = pRgt.substring(partSize).trim();
+    return [pLft, pRgt];
   },
 
   errorInfo(err) {
@@ -129,11 +129,6 @@ const Rag = {
     DataMgr.readDbDocNames();
     DataMgr.readDbDocs();
     this.ragQuery = query;
-    // this.ragContext = "";  //XXX
-    // this.ragResponse = "";
-    // this.responses = [];
-    // this.prompts = [];
-    // ThreadMgr.rows=[];
     let ndoc = 0;
 
     try {
@@ -142,30 +137,24 @@ const Rag = {
         if (doc.trim() == "") continue;
         const doc_name = DataMgr.doc_names[i];
         const doc_entire_len = doc.length;
-        UaLog.log(`${doc_name} (${doc_entire_len}) `);
         xlog(`${doc_name} (${doc_entire_len}) `);
+        UaLog.log(`${doc_name} (${doc_entire_len}) `);
+
         ++ndoc;
         let npart = 0;
-        let partLft = "";
-        let partRgt = "";
         let decr = 0;
         let prompt = "";
-        // etsrazione informazioi da documenti
+        let lft = "";
+        let rgt = "";
+
         while (true) {
           let partSize = this.getPartSize(doc, promptDoc("", query), decr);
-          if (partSize < 10) break;
-          // xlog("====================");
-          // xlog("partLft.lemngth:", partLft.length);
-          // xlog("partRgt.lemngth:", partRgt.length);
-          // xlog("prompt.length;", prompt.length);
-          [partLft, partRgt] = this.getPartDoc(doc, partSize);
-          prompt = promptDoc(partLft, query);
-          // xlog("------------");
-          // xlog("decr:", decr);
-          // xlog("partSize:", partSize);
-          // xlog("partLft.lemngth:", partLft.length);
-          // xlog("partRgt.lemngth:", partRgt.length);
-          // xlog("prompt.length;", prompt.length);
+          if (partSize < 10) {
+            break;
+          }
+          [lft, rgt] = this.getPartDoc(doc, partSize);
+          this.ragLog(`${ndoc},${npart + 1}`, lft.length, rgt.length);
+          prompt = promptDoc(lft, query);
           const payload = getPayloadDoc(prompt);
           let text;
           try {
@@ -174,8 +163,8 @@ const Rag = {
           } catch (err) {
             const e = this.errorInfo(err);
             if (e.errorType === ERROR_TOKENS) {
-              UaLog.log(`Error tokens.1  ${prompt.length}`);
-              xlog(`Error tokens.  ${prompt.length}`);
+              UaLog.log(`Error tokens.1  ${lft.length}`);
+              xerror(`Error tokens.  ${prompt.length}`);
               decr += PROMPT_DECR;
               continue;
             } else {
@@ -184,62 +173,19 @@ const Rag = {
               throw s;
             }
           }
+
           npart++;
-          doc = partRgt;
-          if (npart == 1) {
-            text = `Documento:${doc_name}\n${text}`;
-          }
-          this.responses.push(text);
-          this.log(`${ndoc},${npart}`, partLft.length, partRgt.length);
-        }
-      }
+          doc = rgt;
+          const docText = `<<<${doc_name}>>>\n${text}`; //TODO
+          this.responses.push(docText);
+          // this.ragLog(`${doc_name}  ${ndoc},${npart}`, lft.length, rgt.length);
+        } // end while
+      } // end for
     } catch (error) {
       alert("requestDocsRAG(1)\n" + error);
       xerror(error);
     }
 
-    // elenco risposte accumulate => contesto
-    /*
-    try {
-      let resps = list2text(this.responses);
-      let decr = 0;
-      let text = "";
-      let prompt = "";
-
-      while (true) {
-        resps = this.truncInput(resps, decr);
-        prompt = promptToContext(resps, query);
-        const payload = getPlayloadContext(prompt);
-
-        try {
-          text = await HfRequest.post(payload);
-          if (!text) return "";
-        } catch (err) {
-          const e = this.errorInfo(err);
-          if (e.errorType === ERROR_TOKENS) {
-            UaLog.log(`Error tokens.2  ${prompt.length}`);
-            xlog(`Error tokens.  ${prompt.length}`);
-            decr += PROMPT_DECR;
-            continue;
-          } else {
-            xerror(err);
-            const s = this.errorTotext(err);
-            throw s;
-          }
-        }
-        break;
-      }
-      const pl = prompt.length;
-      // text = cleanContext(text); 
-      this.ragContext = text;
-      this.saveToDb();
-      UaLog.log(`Contesto  (${pl})`);
-    } catch (error) {
-      xerror(error);
-      alert("requestDocsRAG(2)\n" + error);
-      throw error;
-    }
-      */
     const resps = list2text(this.responses);
     this.ragContext = resps;
     this.saveToDb();
@@ -349,7 +295,7 @@ const Rag = {
             const e = this.errorInfo(err);
             if (e.errorType === ERROR_TOKENS) {
               UaLog.log(`Error tokens.5  ${prompt.length}`);
-              xlog(`Error tokens.  ${prompt.length}`);
+              xerror(`Error tokens.  ${prompt.length}`);
               decr += PROMPT_DECR;
               continue;
             } else {

static/javascript/rag_rqs.js

@@ -19,7 +19,7 @@
  */
 
 "use strict";
-const VERS = "0.1.39 (24-07-2024)";
+const VERS = "0.1.40 (28-07-2024)";
 
 var xlog = console.log;
 var xerror = console.error;
@@ -118,7 +118,8 @@ function elencoRisposte(e) {
   if (rs.length == 0) return;
   const text = rs
     .map((x, i) => {
-      return `[${i + 1}]\n ${x.trim()}`;
+      x = subResponseDOcTag(x);
+      return `\n[${i + 1}]\n ${x.trim()}`;
     })
     .join("\n");
   wnds.wpre.show(text);

static/javascript/ualog.js

@@ -64,7 +64,7 @@ var UaLog = {
       maxHeight: "400px",
       overflow: "auto",
       // scrollbarColor: "#ab3205 #ffffff",
-      scrollbarColor: "##027876 #454444",
+      scrollbarColor: "#027876 #454444",
       scrollbarWidth: "auto",
     };
     for (const k in msg_css) msg.style[k] = msg_css[k];

static/js/ragrqs.min.js

@@ -2,7 +2,9 @@ const removeChars=a=>{a=a.replace(/[\u00AD\u200B\u200C\u200D\u2060\uFEFF]/g,"");
 function cleanDoc(a){try{return a=removeTag(a),a=removeChars(a),a=replaceChars(a),a=a.replace(/\n/g," "),a=a.replace(/\t/g," "),a=a.replace(/ +/g," "),a=a.replace(/\n\s*\n/g,"\n"),a=a.replace(/[\u201c\u201d]/g,'"'),a=a.replace(/[\u2018\u2019]/g,"'"),a=a.replace(/[\u00ab\u00bb\u201e\u201c]/g,'"'),a=a.replace(/[\u2013\u2014]/g,"-"),a=a.replace(/\\[nrt]/g,""),a=a.replace(/[^\w\s\u00e0\u00e8\u00e9\u00ec\u00f2\u00f9\u00c0\u00c8\u00c9\u00cc\u00d2\u00d9\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00e4\u00eb\u00ef\u00f6\u00fc\u00c4\u00cb\u00cf\u00d6\u00dc\u00e2\u00ea\u00ee\u00f4\u00fb\u00c2\u00ca\u00ce\u00d4\u00db\u00e7\u00c7\u00f1\u00d1.,;:!?'"()-]/g,
 ""),a.trim()}catch(b){return xerror(b),"Errore di codifica nel documento"}}
 function cleanResponse(a){try{return a=removeChars(a),a=replaceChars(a),a=a.replace(/\t/g," "),a=a.replace(/ +/g," "),a=a.replace(/\n\s*\n/g,"\n"),a=a.replace(/[\u201c\u201d]/g,'"'),a=a.replace(/[\u2018\u2019]/g,"'"),a=a.replace(/[\u00ab\u00bb\u201e\u201c]/g,'"'),a=a.replace(/[\u2013\u2014]/g,"-"),a=a.replace(/\\[nrt]/g,""),a=a.replace(/[^\w\s\u00e0\u00e8\u00e9\u00ec\u00f2\u00f9\u00c0\u00c8\u00c9\u00cc\u00d2\u00d9\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00e4\u00eb\u00ef\u00f6\u00fc\u00c4\u00cb\u00cf\u00d6\u00dc\u00e2\u00ea\u00ee\u00f4\u00fb\u00c2\u00ca\u00ce\u00d4\u00db\u00e7\u00c7\u00f1\u00d1.,;:!?'"()-]/g,""),
-a.trim()}catch(b){return xerror(b),"Errore di codifica nela risposta"}}function list2text(a){return a.map(b=>`${cleanResponse(b)}`).join("\n\n")}function cleanOutput(a){a=a.trim();a=a.replace(/\n2,}/g,"\n");a=a.replace(/^[ \t]+/gm,"");a=a.replace(/^(\s*[-*\u2022])(\s*)/gm,"\n$1 ");a=a.replace(/^(\s*\d+\.)(\s*)/gm,"\n$1 ");a=a.replace(/(```[\s\S]*?```)/g,"\n\n$1\n\n");return a=a.replace(/[ \t]+$/gm,"")};const UaDb={create(a,b){localStorage.getItem(a)?xerror(`ID ${a} already exists.`):localStorage.setItem(a,b)},read(a){const b=localStorage.getItem(a);return null===b?(xlog(`UaDb.read  ${a} not found.`),""):b},update(a,b){localStorage.getItem(a)?localStorage.setItem(a,b):xlog(`UaDb.update ${a} not found.`)},delete(a){localStorage.getItem(a)?localStorage.removeItem(a):xerror(`ID ${a} not found.`)},save(a,b){localStorage.setItem(a,b)},getAllIds(){const a=[];for(let b=0;b<localStorage.length;b++)a.push(localStorage.key(b));
+a.trim()}catch(b){return xerror(b),"Errore di codifica nela risposta"}}
+function cleanOutput(a){try{return a=removeChars(a),a=replaceChars(a),a=a.replace(/\t/g," "),a=a.replace(/ +/g," "),a=a.replace(/\n\s*\n/g,"\n"),a=a.replace(/[\u201c\u201d]/g,'"'),a=a.replace(/[\u2018\u2019]/g,"'"),a=a.replace(/[\u00ab\u00bb\u201e\u201c]/g,'"'),a=a.replace(/[\u2013\u2014]/g,"-"),a=a.replace(/\\[nrt]/g,""),a=a.replace(/[^\w\s\u00e0\u00e8\u00e9\u00ec\u00f2\u00f9\u00c0\u00c8\u00c9\u00cc\u00d2\u00d9\u00e1\u00e9\u00ed\u00f3\u00fa\u00c1\u00c9\u00cd\u00d3\u00da\u00e4\u00eb\u00ef\u00f6\u00fc\u00c4\u00cb\u00cf\u00d6\u00dc\u00e2\u00ea\u00ee\u00f4\u00fb\u00c2\u00ca\u00ce\u00d4\u00db\u00e7\u00c7\u00f1\u00d1.,;:!?'"()-]/g,""),
+a.trim()}catch(b){return xerror(b),"Errore di codifica nell'output"}}function list2text(a){const b={};for(var c of a){a=(a=c.match(/<<<(.*?)>>>/))?a[1]:null;const d=c.replace(/<<<.*?>>>/,"");b[a]=a in b?b[a]+`\n ${d}`:d}c=[];for(let [d,e]of Object.entries(b))c.push(`Documento: ${d}.\n${e}`);return c.join("\n\n")}function subResponseDOcTag(a){return a.replace(/<<<(.*?)>>>/,(b,c)=>`Documento: ${c}`)};const UaDb={create(a,b){localStorage.getItem(a)?xerror(`ID ${a} already exists.`):localStorage.setItem(a,b)},read(a){const b=localStorage.getItem(a);return null===b?(xlog(`UaDb.read  ${a} not found.`),""):b},update(a,b){localStorage.getItem(a)?localStorage.setItem(a,b):xlog(`UaDb.update ${a} not found.`)},delete(a){localStorage.getItem(a)?localStorage.removeItem(a):xerror(`ID ${a} not found.`)},save(a,b){localStorage.setItem(a,b)},getAllIds(){const a=[];for(let b=0;b<localStorage.length;b++)a.push(localStorage.key(b));
 return a},saveArray(a,b){b=JSON.stringify(b);UaDb.save(a,b)},readArray(a){a=UaDb.read(a);return 0==a.trim().length?[]:JSON.parse(a)},saveJson(a,b){b=JSON.stringify(b);UaDb.save(a,b)},readJson(a){return(a=UaDb.read(a))?JSON.parse(a):{}}},DataMgr={docs:[],doc_names:[],linkToName(a){a=a.split("/");return a[a.length-1]},async loadDoc(a){try{const b=await requestGet(a),c=cleanDoc(b),d=this.linkToName(a);this.doc_names.push(d);this.docs.push(c);this.saveDbDocs();return c}catch(b){alert("loadDoc()\n"+b+
 "\n"+a)}},addDoc(a,b){b=cleanDoc(b);this.docs.push(b);this.doc_names.push(a);this.saveDbDocs()},saveDbDocs(){UaDb.saveArray("id_doc_names",this.doc_names);UaDb.saveArray("id_docs",this.docs)},readDbDocs(){this.docs=UaDb.readArray("id_docs")},readDbDocNames(){this.doc_names=UaDb.readArray("id_doc_names")},deleteJsonDati(){const a=UaDb.getAllIds();for(const b of a)["id_docs","id_doc_names"].includes(b)||UaDb.delete(b);Rag.ragQuery="";Rag.ragContext="";Rag.ragResponse="";Rag.responses=[];Rag.prompts=
 [];ThreadMgr.rows=[]}};const arr=["bWtkW0l+XX0=","SXx2d1FxbVc=","V1tRXlxneUY=","flV6a1NdbUg=","VllpUkc="],model_name="mistralai/Mistral-7B-Instruct-v0.3",umgm=a=>a.map(b=>atob(b).split("").map(c=>String.fromCharCode((c.charCodeAt(0)-5+256)%256)).join("")).join(""),tm=umgm(arr);
@@ -27,18 +29,18 @@ const e=await d.json();if(!d.ok){const g=this.checkError(d.status,e),l=this.crea
  You should have received a copy of the GNU General Public License
  along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
-const MAX_PROMPT_LENGTH=81920,PROMPT_DECR=2048,Rag={ragContext:"",ragQuery:"",ragResponse:"",responses:[],prompts:[],doc:"",doc_part:"",init(){this.readRespsFromDb();this.readFromDb()},saveToDb(){UaDb.saveJson("id_rag",{context:this.ragContext,ragquery:this.ragQuery,ragresponse:this.ragResponse});UaDb.saveArray("id_thread",ThreadMgr.rows)},readFromDb(){const a=UaDb.readJson("id_rag");this.ragContext=a.context||"";this.ragQuery=a.ragquery||"";this.ragResponse=a.ragresponse||"";ThreadMgr.rows=UaDb.readArray("id_thread")},
-saveRespToDb(){UaDb.saveArray("id_responses",this.responses)},readRespsFromDb(){this.responses=UaDb.readArray("id_responses")},addPrompt(a){this.prompts.push(a)},responsesLength(){return this.responses.reduce((a,b)=>a+b.length,0)},log(a,b,c){const d=MAX_PROMPT_LENGTH,e=this.responsesLength();xlog(`${a} mx:${d} lft:${b} rgt:${c} arr:${e}`);UaLog.log(`${a}${"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"}${b}${"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"}${c}${"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"}${e}`)},truncInput(a,
+const MAX_PROMPT_LENGTH=87040,PROMPT_DECR=2048,Rag={ragContext:"",ragQuery:"",ragResponse:"",responses:[],prompts:[],doc:"",doc_part:"",init(){this.readRespsFromDb();this.readFromDb()},saveToDb(){UaDb.saveJson("id_rag",{context:this.ragContext,ragquery:this.ragQuery,ragresponse:this.ragResponse});UaDb.saveArray("id_thread",ThreadMgr.rows)},readFromDb(){const a=UaDb.readJson("id_rag");this.ragContext=a.context||"";this.ragQuery=a.ragquery||"";this.ragResponse=a.ragresponse||"";ThreadMgr.rows=UaDb.readArray("id_thread")},
+saveRespToDb(){UaDb.saveArray("id_responses",this.responses)},readRespsFromDb(){this.responses=UaDb.readArray("id_responses")},addPrompt(a){this.prompts.push(a)},responsesLength(){return this.responses.reduce((a,b)=>a+b.length,0)},ragLog(a,b,c){const d=MAX_PROMPT_LENGTH,e=this.responsesLength();xlog(`${a} mx:${d} lft:${b} rgt:${c} arr:${e}`);UaLog.log(`${a}${"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"}${b}${"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"}${c}${"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"}${e}`)},truncInput(a,
 b){return a.substring(0,a.length-b)},getPartSize(a,b,c){c=MAX_PROMPT_LENGTH-c;a.length+b.length<c?c=a.length:(a=a.indexOf(".",c),a=(-1!=a?a:c)+1,a>c+100&&(a=c),c=a);return c},getPartDoc(a,b){const c=a.substring(0,b);a=a.substring(b).trim();return[c,a]},errorInfo(a){return JSON.parse(a.message)},errorTotext(a){a=JSON.parse(a.message);return`Errore:
     Stato: ${a.status}
     Descrizione stato: ${a.statusText}
     Tipo di errore: ${a.errorType}
-    Messaggio: ${a.message}`},async requestDocsRAG(a){DataMgr.deleteJsonDati();DataMgr.readDbDocNames();DataMgr.readDbDocs();this.ragQuery=a;var b=0;try{for(let c=0;c<DataMgr.docs.length;c++){let d=DataMgr.docs[c];if(""==d.trim())continue;const e=DataMgr.doc_names[c],f=d.length;UaLog.log(`${e} (${f}) `);xlog(`${e} (${f}) `);++b;let g=0,l="",h="",k=0,n="";for(;;){let q=this.getPartSize(d,promptDoc("",a),k);if(10>q)break;[l,h]=this.getPartDoc(d,q);n=promptDoc(l,a);const r=getPayloadDoc(n);let m;try{if(m=
-await HfRequest.post(r),!m)return""}catch(p){if(this.errorInfo(p).errorType===ERROR_TOKENS){UaLog.log(`Error tokens.1  ${n.length}`);xlog(`Error tokens.  ${n.length}`);k+=PROMPT_DECR;continue}else throw xerror(p),this.errorTotext(p);}g++;d=h;1==g&&(m=`Documento:${e}\n${m}`);this.responses.push(m);this.log(`${b},${g}`,l.length,h.length)}}}catch(c){alert("requestDocsRAG(1)\n"+c),xerror(c)}this.ragContext=list2text(this.responses);this.saveToDb();b="";try{let c=promptWithContext(this.ragContext,a);const d=
-getPayloadWithContext(c);try{if(b=await HfRequest.post(d),!b)return""}catch(e){this.errorInfo(e);const f=this.errorTotext(e);xerror(e);throw f;}this.ragResponse=b;this.saveRespToDb();ThreadMgr.init();this.saveToDb();UaLog.log(`Risposta  (${c.length},${b.length})`)}catch(c){throw b=c,xerror(c),alert("requestDocsRAG(3)\n"+c),c;}finally{return b=cleanOutput(b)}},async requestContext(a){if(!this.ragContext&&!await confirm("Contesto vuoto. Vuoi continuare?"))return"";if(ThreadMgr.isFirst()){var b="";ThreadMgr.init();
-try{var c=this.ragContext,d=ThreadMgr.getThread();prompt=promptThread(c,d,a);var e=getPayloadThread(prompt);try{var f=await HfRequest.post(e);if(!f)return""}catch(g){throw this.errorInfo(g).errorType===ERROR_TOKENS?UaLog.log(`Error tokens.4  ${prompt.length}`):alert(g),xerror(g),g;}ThreadMgr.add(a,f);b=ThreadMgr.getOutText();UaLog.log(`Inizio Conversazione (${prompt.length},${f.length})`)}catch(g){throw xerror(g),b=g,g;}finally{return b=cleanOutput(b)}}else{c="";try{b=this.ragContext;let g=ThreadMgr.getThread();
-e=d="";for(f=0;;){g=this.truncInput(g,f);d=promptThread(b,g,a);const l=getPayloadThread(d);try{if(e=await HfRequest.post(l),!e)return""}catch(h){const k=this.errorInfo(h);if(k.errorType===ERROR_TOKENS){UaLog.log(`Error tokens.5  ${d.length}`);xlog(`Error tokens.  ${d.length}`);f+=PROMPT_DECR;continue}else throw xerror(h),k.errorType;}break}ThreadMgr.add(a,e);c=ThreadMgr.getOutText();UaLog.log(`Conversazione  (${d.length},${e.length})`)}catch(g){throw alert("requestContext(2) \n"+g),xerror(g),c=g,
-g;}finally{return c=cleanOutput(c)}}}},LLM="Assistant:",USER="User:",ThreadMgr={rows:[],init(){this.rows=[];Rag.ragResponse?this.add(Rag.ragQuery,Rag.ragResponse):this.add("","")},add(a,b){this.rows.push([a,b]);UaDb.saveArray("id_thread",ThreadMgr.rows)},getOutText(){const a=[];for(const b of this.rows){const c=b[0].trim(),d=b[1].trim();a.push(`\n${USER}\n${c}\n${LLM}\n${d}\n\n`)}return a.join("").trim()},getThread(){const a=[];for(const b of this.rows){const c=b[0].trim(),d=b[1].trim();a.push(`${USER}\n${c}\n${LLM}\n${d}\n\n`)}return a.join("").trim()},
+    Messaggio: ${a.message}`},async requestDocsRAG(a){DataMgr.deleteJsonDati();DataMgr.readDbDocNames();DataMgr.readDbDocs();this.ragQuery=a;var b=0;try{for(let c=0;c<DataMgr.docs.length;c++){let d=DataMgr.docs[c];if(""==d.trim())continue;const e=DataMgr.doc_names[c],f=d.length;xlog(`${e} (${f}) `);UaLog.log(`${e} (${f}) `);++b;let g=0,l=0,h="",k="",m="";for(;;){let q=this.getPartSize(d,promptDoc("",a),l);if(10>q)break;[k,m]=this.getPartDoc(d,q);this.ragLog(`${b},${g+1}`,k.length,m.length);h=promptDoc(k,
+a);const r=getPayloadDoc(h);let n;try{if(n=await HfRequest.post(r),!n)return""}catch(p){if(this.errorInfo(p).errorType===ERROR_TOKENS){UaLog.log(`Error tokens.1  ${k.length}`);xerror(`Error tokens.  ${h.length}`);l+=PROMPT_DECR;continue}else throw xerror(p),this.errorTotext(p);}g++;d=m;this.responses.push(`<<<${e}>>>\n${n}`)}}}catch(c){alert("requestDocsRAG(1)\n"+c),xerror(c)}this.ragContext=list2text(this.responses);this.saveToDb();b="";try{let c=promptWithContext(this.ragContext,a);const d=getPayloadWithContext(c);
+try{if(b=await HfRequest.post(d),!b)return""}catch(e){this.errorInfo(e);const f=this.errorTotext(e);xerror(e);throw f;}this.ragResponse=b;this.saveRespToDb();ThreadMgr.init();this.saveToDb();UaLog.log(`Risposta  (${c.length},${b.length})`)}catch(c){throw b=c,xerror(c),alert("requestDocsRAG(3)\n"+c),c;}finally{return b=cleanOutput(b)}},async requestContext(a){if(!this.ragContext&&!await confirm("Contesto vuoto. Vuoi continuare?"))return"";if(ThreadMgr.isFirst()){var b="";ThreadMgr.init();try{var c=
+this.ragContext,d=ThreadMgr.getThread();prompt=promptThread(c,d,a);var e=getPayloadThread(prompt);try{var f=await HfRequest.post(e);if(!f)return""}catch(g){throw this.errorInfo(g).errorType===ERROR_TOKENS?UaLog.log(`Error tokens.4  ${prompt.length}`):alert(g),xerror(g),g;}ThreadMgr.add(a,f);b=ThreadMgr.getOutText();UaLog.log(`Inizio Conversazione (${prompt.length},${f.length})`)}catch(g){throw xerror(g),b=g,g;}finally{return b=cleanOutput(b)}}else{c="";try{b=this.ragContext;let g=ThreadMgr.getThread();
+e=d="";for(f=0;;){g=this.truncInput(g,f);d=promptThread(b,g,a);const l=getPayloadThread(d);try{if(e=await HfRequest.post(l),!e)return""}catch(h){const k=this.errorInfo(h);if(k.errorType===ERROR_TOKENS){UaLog.log(`Error tokens.5  ${d.length}`);xerror(`Error tokens.  ${d.length}`);f+=PROMPT_DECR;continue}else throw xerror(h),k.errorType;}break}ThreadMgr.add(a,e);c=ThreadMgr.getOutText();UaLog.log(`Conversazione  (${d.length},${e.length})`)}catch(g){throw alert("requestContext(2) \n"+g),xerror(g),c=
+g,g;}finally{return c=cleanOutput(c)}}}},LLM="Assistant:",USER="User:",ThreadMgr={rows:[],init(){this.rows=[];Rag.ragResponse?this.add(Rag.ragQuery,Rag.ragResponse):this.add("","")},add(a,b){this.rows.push([a,b]);UaDb.saveArray("id_thread",ThreadMgr.rows)},getOutText(){const a=[];for(const b of this.rows){const c=b[0].trim(),d=b[1].trim();a.push(`\n${USER}\n${c}\n${LLM}\n${d}\n\n`)}return a.join("").trim()},getThread(){const a=[];for(const b of this.rows){const c=b[0].trim(),d=b[1].trim();a.push(`${USER}\n${c}\n${LLM}\n${d}\n\n`)}return a.join("").trim()},
 isFirst(){return 2>this.rows.length}};function promptDoc(a,b,c){return`
 SYSTEM:
 Sei un assistente AI specializzato nell'analisi di documenti. Rispondi sempre in italiano.
@@ -139,9 +141,10 @@ OUTPUT_FORMAT:
 RESPONSE:
 `}function getPayloadDoc(a){return{inputs:a,parameters:{task:"text2text-generation",max_new_tokens:512,num_return_sequences:1,temperature:.5,top_k:50,top_p:.7,do_sample:!0,no_repeat_ngram_size:4,num_beams:5,repetition_penalty:1.4,return_full_text:!1,details:!1,max_time:90,seed:42},options:{use_cache:!1,wait_for_model:!0}}}
 function getPayloadWithContext(a){return{inputs:a,parameters:{task:"text-generation",max_new_tokens:1024,num_return_sequences:1,temperature:.6,top_k:50,top_p:.7,do_sample:!0,no_repeat_ngram_size:4,num_beams:5,repetition_penalty:1.4,return_full_text:!1,details:!1,max_time:120,seed:42},options:{use_cache:!1,wait_for_model:!0}}}
-function getPayloadThread(a){return{inputs:a,parameters:{task:"text-generation",max_new_tokens:1024,num_return_sequences:1,temperature:.6,top_k:50,top_p:.7,do_sample:!0,no_repeat_ngram_size:43,num_beams:5,repetition_penalty:1.4,return_full_text:!1,details:!1,max_time:120,seed:42},options:{use_cache:!1,wait_for_model:!0}}};const VERS="0.1.39 (24-07-2024)";var xlog=console.log,xerror=console.error;const cancelRequest=()=>{confirm("Confermi Cancellazione Richeista ?")&&(HfRequest.cancelRequest(),hideSpinner())},showSpinner=()=>{const a=document.getElementById("spinner");a.classList.add("show-spinner");a.addEventListener("click",cancelRequest)},hideSpinner=()=>{const a=document.getElementById("spinner");a.classList.remove("show-spinner");a.removeEventListener("click",cancelRequest)};
+function getPayloadThread(a){return{inputs:a,parameters:{task:"text-generation",max_new_tokens:1024,num_return_sequences:1,temperature:.6,top_k:50,top_p:.7,do_sample:!0,no_repeat_ngram_size:43,num_beams:5,repetition_penalty:1.4,return_full_text:!1,details:!1,max_time:120,seed:42},options:{use_cache:!1,wait_for_model:!0}}};const VERS="0.1.40 (28-07-2024)";var xlog=console.log,xerror=console.error;const cancelRequest=()=>{confirm("Confermi Cancellazione Richeista ?")&&(HfRequest.cancelRequest(),hideSpinner())},showSpinner=()=>{const a=document.getElementById("spinner");a.classList.add("show-spinner");a.addEventListener("click",cancelRequest)},hideSpinner=()=>{const a=document.getElementById("spinner");a.classList.remove("show-spinner");a.removeEventListener("click",cancelRequest)};
 function openApp(){wnds.init();Menu.init();TextInput.init();TextOutput.init();Rag.init();document.querySelector(".menu-btn").checked=!1;release()}function release(){document.querySelector(".release").innerHTML=VERS}const op0=async function(a){a=await requestGet("./help1.html");wnds.wdiv.show(a)};function showQuery(a){wnds.wpre.show(`\n${Rag.ragQuery}`)}function showRagResponse(a){wnds.wpre.show(`\n${Rag.ragResponse}`)}function showThread(a){a=ThreadMgr.getOutText();wnds.wpre.show(a)}
-function elencoRisposte(a){a=[...Rag.responses];0==a.length&&(a=UaDb.readArray("id_responses"));0!=a.length&&(a=a.map((b,c)=>`[${c+1}]\n ${b.trim()}`).join("\n"),wnds.wpre.show(a))}function showContesto(a){wnds.wpre.show(`${Rag.ragContext}`)}function elencoDati(a){var b=UaDb.getAllIds();a=[];for(var c of b)b=UaDb.read(c).length,a.push(`${c} (${b})`);c=a.join("\n ");wnds.wpre.show(c)}async function elencoDocs(a){DataMgr.readDbDocNames();a=DataMgr.doc_names.join("\n");wnds.wpre.show(a)}
+function elencoRisposte(a){a=[...Rag.responses];0==a.length&&(a=UaDb.readArray("id_responses"));0!=a.length&&(a=a.map((b,c)=>{b=subResponseDOcTag(b);return`\n[${c+1}]\n ${b.trim()}`}).join("\n"),wnds.wpre.show(a))}function showContesto(a){wnds.wpre.show(`${Rag.ragContext}`)}function elencoDati(a){var b=UaDb.getAllIds();a=[];for(var c of b)b=UaDb.read(c).length,a.push(`${c} (${b})`);c=a.join("\n ");wnds.wpre.show(c)}
+async function elencoDocs(a){DataMgr.readDbDocNames();a=DataMgr.doc_names.join("\n");wnds.wpre.show(a)}
 function calcQuery(){DataMgr.readDbDocs();DataMgr.readDbDocNames();var a=[];let b=0,c=0;a.push("Documento   Num.Parti");a.push("==================");for(const d of DataMgr.docs){const e=DataMgr.doc_names[c];c+=1;const f=Math.ceil(d.length/MAX_PROMPT_LENGTH);b+=f;a.push(`${e}&nbsp;&nbsp;&nbsp;[${f}]`)}a.push("==================");a.push(`Totale num. Parti: ${b}`);a=a.join("\n");wnds.wpre.show(a)}
 function deleteDati(a){confirm("Confermi cancellazione dati?")&&(DataMgr.deleteJsonDati(),wnds.wdiv.close(),wnds.wpre.close(),TextOutput.clear())}function deleteSttorage(a){confirm("Confermi cancellazione documenti & dati?")&&(DataMgr.deleteJsonDati(),localStorage.clear(),wnds.wdiv.close(),wnds.wpre.close(),TextOutput.clear(),DataMgr.docs=[],DataMgr.doc_names=[])}async function help1(a){a=await requestGet("./data/help_test.html");wnds.wdiv.show(a)}
 function loadTestoEsempio(a){DataMgr.loadDoc(`data/${a}`);wnds.wdiv.close()}async function help2(a){a=await requestGet("./help2.html");wnds.wdiv.show(a)}function showPrompts(a){0!=Rag.prompts.length&&(a=Rag.prompts.map((b,c)=>`[${c+1}]${b}\n`).join("\n"),wnds.wpre.show(a))}function showTesti(a){DataMgr.readDbDocs();a=DataMgr.docs.join("\n");wnds.wpre.show(a)};const WndPre=a=>({w:UaWindowAdm.create(a),out:null,show(b){wnds.closeAll();b=`
@@ -186,7 +189,7 @@ document.body.appendChild(this.workerScriptElement);this.workerScriptElement.onl
 async function readTextFile(a){if(!a||"text/plain"!==a.type)throw Error("Invalid file type. Please select a text file.");return new Promise((b,c)=>{const d=new FileReader;d.onload=e=>b(e.target.result);d.onerror=e=>c(Error("Error reading file: "+e.message));d.readAsText(a)})};const nodrag_tds=["input","select","a"],nodrag_cls="nodrag";
 var UaDrag=function(a){return function(b){let c=0,d=0,e=0,f=0;const g=function(h){h=h||window.event;h.preventDefault();c=e-h.clientX;d=f-h.clientY;e=h.clientX;f=h.clientY;b.style.top=b.offsetTop-d+"px";b.style.left=b.offsetLeft-c+"px"},l=function(){document.onmouseup=null;document.onmousemove=null};b.onmousedown=function(h){h=h||window.event;let k=h.target;!(k=k||null)||nodrag_tds.includes(k.tagName.toLowerCase())||k.classList.contains(nodrag_cls)||(h.preventDefault(),e=h.clientX,f=h.clientY,document.onmouseup=
 l,document.onmousemove=g)}}(a)};const UaJtfh=()=>({rows:[],init(){this.rows=[];return this},insert(a){this.rows.unshift(a);return this},append(a){this.rows.push(a);return this},text(a=""){return this.rows.join(a)},html(a=""){return this.rows.join(a).replace(/\s+|\[rn\]/g," ")}});var UaLog={callHide:function(){},callShow:function(){},active:!1,wind:null,x:null,y:null,z:null,max_length:2E3,msg_id:"ualogmsg_",new:function(){null==this.wind&&(this.wind=UaWindowAdm.create("ualog_"),this.wind.drag());this.wind.setHtml('\n         <button type="button" onclick="javascript:UaLog.cls();">Clear</button>\n         <button type="button" onclick="javascript:UaLog.close();">Close</button>\n         <div id="ualogmsg_"></div>');this.wind.setStyle({width:"auto",minWidth:"300px",maxWidth:"400px",
-height:"auto",textAlign:"center",padding:"2px 2px 2px 2px",margin:"5px 0 0 0",background:"#333333",color:"#ffffff",fontSize:"15px",fontWeight:"normal",borderRadius:"9px",border:"1px solid #999999"});var a=document.getElementById(this.msg_id);const b={width:"auto",textAlign:"left",fontSize:"16px",fontFamily:"monospace",paddingTop:"2px ",marginTop:"2px",color:"#ffffff",background:"#000000",maxHeight:"400px",overflow:"auto",scrollbarColor:"##027876 #454444",scrollbarWidth:"auto"};for(const d in b)a.style[d]=
+height:"auto",textAlign:"center",padding:"2px 2px 2px 2px",margin:"5px 0 0 0",background:"#333333",color:"#ffffff",fontSize:"15px",fontWeight:"normal",borderRadius:"9px",border:"1px solid #999999"});var a=document.getElementById(this.msg_id);const b={width:"auto",textAlign:"left",fontSize:"16px",fontFamily:"monospace",paddingTop:"2px ",marginTop:"2px",color:"#ffffff",background:"#000000",maxHeight:"400px",overflow:"auto",scrollbarColor:"#027876 #454444",scrollbarWidth:"auto"};for(const d in b)a.style[d]=
 b[d];const c={background:"#444444",color:"#ffffff",padding:"5px 5px 5px 5px",margin:"0 5px 0 5px",fontSize:"16px",fontWeight:"bold",border:"1px solid #ffffff",borderRadius:"10px"};a=document.querySelectorAll("#ualog_  button");for(const d of a){for(const e in c)d.style[e]=c[e];d.addEventListener("mouseover",e=>{e.target.style.cursor="pointer";e.target.style.color="#000000";e.target.style.background="#aaaaaa"});d.addEventListener("mouseout",e=>{for(const f in c)e.target.style[f]=c[f]})}this.x?this.wind.vw_vh().setXY(this.x,
 this.y,-1):this.wind.setCenter(-1);this.z&&this.wind.setZ(this.z);return this},setXY(a,b){this.x=a;this.y=b;return this},setZ(a){this.z=a;return this},prn_(...a){a=a.join("<br/>");let b=document.getElementById(this.msg_id);b.innerHTML=b.innerHTML+a+"<br/>"},print(...a){null!=this.wind&&this.active&&this.prn_(...a)},log(...a){null!=this.wind&&this.prn_(...a)},log_show(...a){null!=this.wind&&(this.active||this.toggle(),this.prn_(...a))},cls(){if(null!=this.wind)return document.getElementById(this.msg_id).innerHTML=
 "",this},close(){null!=this.wind&&(this.wind.hide(),this.active=!1,this.callHide())},toggle(){null!=this.wind&&(this.active?(this.active=!1,this.wind.hide(),this.callHide()):(this.active=!0,this.wind.show(),this.callShow()))}};var UaWindowAdm={ws:{},create(a,b=null){let c=document.getElementById(a);c||(c=document.createElement("div"),b?document.getElementById(b).appendChild(c):document.body.appendChild(c),c.id=a,c.setAttribute("data-name","ua-window"),b=this.newUaWindow(c),this.ws[a]=b);a=this.ws[a];c.style.display="none";return a},get(a){return this.ws[a]?this.ws[a]:null},show(a){this.ws[a]&&this.ws[a].show()},close(a){this.ws[a]&&this.ws[a].close()},toggle(a){this.ws[a]&&this.ws[a].toggle()},hide(a){this.ws[a]&&this.ws[a].hide()},