i have been creating a application which is connected to the drive from there they fetch the .docx file then i have converted the docx file into the html format so that LLM easily understand where is heading in the document but the problem i am facing is chunk creating
i tried to create chunk on the basis of bold formatting tags in the html file
for example such type of tag i consider for making chunks
strong html tag Buyer Management Procedure /strong
but the problem i faced is that when i got improved chunks from openai llm they gave me repeated content which i dont want
i tried everything but nothing works now i am clueless how to solve this probelm
CODE FOR CRREATING CHUNK
function processDocument(htmlContent) {
console.log("Starting document processing...");
const chunks = [];
let customPrompts = {};
// Process custom prompts across the entire document
const processedContent = htmlContent.replace(/(([ws]+))((([ws]+)))/g, (match, text, prompt) => {
customPrompts[text] = prompt;
console.log(`Found custom prompt: "${text}" with instruction "${prompt}"`);
return `(${text})`;
});
const dom = new JSDOM(processedContent);
const document = dom.window.document;
// Function to check if an element is a bold formatting tag
function isBoldTag(element) {
return ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'STRONG', 'B'].includes(element.tagName);
}
// Function to get heading level
function getHeadingLevel(element) {
if (element.tagName.startsWith('H')) {
return parseInt(element.tagName.slice(1));
}
return 0; // For <strong> and <b> tags
}
let currentChunk = [];
let currentHeading = '';
let currentLevel = 0;
function processNode(node) {
if (node.nodeType === Node.ELEMENT_NODE) {
if (isBoldTag(node)) {
// If we have a current chunk, save it
if (currentChunk.length > 0) {
chunks.push({
content: currentChunk.join(''),
heading: currentHeading,
level: currentLevel,
customPrompts: { ...customPrompts }
});
currentChunk = [];
}
currentHeading = node.textContent;
currentLevel = getHeadingLevel(node);
currentChunk.push(node.outerHTML);
} else {
// For non-bold tags, just add their HTML to the current chunk
currentChunk.push(node.outerHTML);
}
// Process child nodes
node.childNodes.forEach(processNode);
} else if (node.nodeType === Node.TEXT_NODE) {
// Add text nodes to the current chunk
currentChunk.push(node.textContent);
}
}
// Start processing from the body
processNode(document.body);
// Add the last chunk if there's any content left
if (currentChunk.length > 0) {
chunks.push({
content: currentChunk.join(''),
heading: currentHeading,
level: currentLevel,
customPrompts: { ...customPrompts }
});
}
console.log("Document processing complete.");
console.log(`Total chunks created: ${chunks.length}`);
// Log all chunks for visibility
chunks.forEach((chunk, index) => {
console.log(`nChunk ${index + 1}:`);
console.log("Heading:", chunk.heading);
console.log("Level:", chunk.level);
console.log("Content:");
console.log(chunk.content);
console.log("Custom prompts:", chunk.customPrompts);
console.log("Word count:", chunk.content.replace(/<[^>]*>/g, '').split(/s+/).filter(Boolean).length);
console.log("-".repeat(50)); // Separator for readability
});
return chunks;
}
// Function to create an OpenAI assistant
async function createAssistant() {
const assistant = await openai.beta.assistants.create({
name: "Document Improvement Assistant",
instructions:
"You are an AI assistant that helps improve documents. Use the existing knowledge base to improve the content provided. Don't use any content from internet or your own knowledge. Enhance clarity, coherence, and relevance of the text. Use proper formatting and heading and make sure such text is improved. Don't add image placeholders or use links. Use simple text and don't use complex formatting. Use formatting which is good looking and readable, and provide detailed information.",
model: "gpt-4o-mini",
tools: [{ type: "file_search" }],
tool_resources: {
file_search: {
vector_store_ids: [VECTOR_STORE_ID],
},
},
});
console.log(`Assistant created with ID: ${assistant.id}`);
return assistant;
}
IMPROVMENT CHUNK CODE
async function improveChunk(chunk, fullDocument, assistantId, documentContext, customPromptInstruction, chunkIndex, totalChunks) {
console.log(Processing chunk ${chunkIndex + 1} of ${totalChunks});
const startTime = Date.now();
try {
const { content, customPrompts } = chunk;
let promptContent = `You are tasked with improving a chunk of text from a larger document. You will be provided with the following information:
-
Document Context:
${documentContext}
-
Custom Prompt Instruction:
${customPromptInstruction}
-
The full document content:
${fullDocument}
-
The current chunk of text to improve (chunk ${chunkIndex + 1} of ${totalChunks}):
<current_chunk>${content}</current_chunk>
Your task is to improve the current chunk of text while ensuring continuity with the rest of the document and staying within the context. Follow these guidelines:
- It is CRUCIAL to avoid ANY repetition of information that exists elsewhere in the document. If you encounter content that appears elsewhere, you MUST either:
a) Omit it to avoid repeatation
- Follow the custom prompt instruction provided above.
- If it’s a heading, format it correctly using Markdown syntax (e.g., # for main headings, ## for subheadings).
- Use Markdown formatting that is visually appealing and readable.
- Ensure smooth transitions with the surrounding content in the full document.
- also donot add such line This revision maintains continuity with the rest of the document while enhancing clarity and readability. The content has been organized into relevant categories, emphasizing the importance of buyers without introducing redundant information.
- if (#,*,##) any line start with such markdown syntax so elaborate that part
Additionally, there are specific parts of the text that require special attention. For each of these parts, enclosed in parentheses (), apply the corresponding custom prompt:
`;
for (const [text, prompt] of Object.entries(customPrompts)) {
promptContent += `For the text "${text}": ${prompt}n`;
}
promptContent += `
Please provide the improved version of the current chunk, or indicate if it should be omitted due to redundancy:`;
const thread = await openai.beta.threads.create({
messages: [{ role: "user", content: promptContent }],
tool_resources: {
file_search: { vector_store_ids: [VECTOR_STORE_ID] },
},
});
const run = await openai.beta.threads.runs.create(thread.id, {
assistant_id: assistantId,
});
let runStatus;
do {
runStatus = await openai.beta.threads.runs.retrieve(thread.id, run.id);
await new Promise((resolve) => setTimeout(resolve, 1000));
} while (runStatus.status !== "completed");
const messages = await openai.beta.threads.messages.list(thread.id);
const improvedChunk = messages.data[0].content[0].text.value;
if (improvedChunk.toLowerCase().includes("this chunk should be omitted")) {
console.log(`Chunk ${chunkIndex + 1} suggested for omission due to redundancy.`);
return null;
}
const endTime = Date.now();
const timeTaken = endTime - startTime;
const processData = new ProcessData({
iterationNumber: fetchCounter,
functionType: 'improve',
chunk: content,
aiPrompt: promptContent,
openaiResponse: improvedChunk,
timeTaken: timeTaken
});
await processData.save();
return improvedChunk;
} catch (error) {
console.error(Error improving chunk:, error);
return chunk.content;
}
}
