I’ve been building a project of creating a webpage design from an image of that page. Most of the application for this purpose uses AI but I am not using them. I went for a complete raw approach with computer vision. I detected text from image using tesseract with its bboxes (bounding boxes – spatial data like {x0, y0, x1, y1} – Here {x0, y0} is top left pixel coodinate and {x1, y1} is the bottom right), I then inpainted the text from the image, used sobel algorithm to detect edges and found their bboxes. It’s not perfect but has worked till here, I then arranged these datas in the proper parent-child heirarchy as json data. Now I only need to arrange this data as html.
Here is an example of the JSON data – heirarchyTree:
[
{
"element": "div",
"text": "",
"bbox": { "x0": 0, "y0": 0, "x1": 1459, "y1": 1091 },
"color": "",
"bgColor": { "r": 0, "g": 0, "b": 0, "a": 0 },
"children": [
{
"element": "div",
"text": "",
"bbox": { "x0": 13, "y0": 13, "x1": 1447, "y1": 869 },
"color": "",
"bgColor": { "r": 255, "g": 255, "b": 255, "a": 255 },
"children": [
{
"element": "p",
"text": "24" Infinora",
"bbox": { "x0": 591, "y0": 29, "x1": 865, "y1": 72 },
"color": { "r": 0, "g": 0, "b": 0, "a": 255 },
"bgColor": "",
"children": [
{
"element": "p",
"text": "4",
"bbox": { "x0": 739, "y0": 30, "x1": 749, "y1": 39 },
"color": { "r": 255, "g": 255, "b": 255, "a": 255 },
"bgColor": "",
"children": []
}
]
},
{
"element": "div",
"text": "",
"bbox": { "x0": 598, "y0": 55, "x1": 632, "y1": 94 },
"color": "",
"bgColor": { "r": 107, "g": 107, "b": 107, "a": 255 },
"children": []
},
....
First I arranged them using position, which works but I can’t move forward with that. No one using an image to html convertor wants their html to be div’s and p tags in the same level arranged inside a single parent div using position absolute right. What I tried to do was first arrange these in an order by how we humans design a component. I tried to come up with an ordering method where the basic idea is we find the components with the smallest x0 and other with smallest y0 which are more smaller y and x respectively. Then I compare the y1 of the one with the smallest y0 and y0 of the one with the smallest x0 to see which has more priority and choose it. Based on this I arranged the children recursively. Then I used another function to give it a layout direction as, if a child has x0 greater than the previous child in order he is in a row else column (for flex based arrangement). Then I wrote a generateCode function. I will provide these functions below
function sortChildrenByDirection(contourTreeNode) {
if (!contourTreeNode.children || contourTreeNode.children.length === 0) {
return;
}
contourTreeNode.children.sort((a, b) => {
const aBox = a.bbox;
const bBox = b.bbox;
const lowestY0 = Math.min(aBox.y0, bBox.y0);
const isATopmost = aBox.y0 === lowestY0;
const isBTopmost = bBox.y0 === lowestY0;
const lowestX0 = Math.min(aBox.x0, bBox.x0);
const isALeftmost = aBox.x0 === lowestX0;
const isBLeftmost = bBox.x0 === lowestX0;
if (aBox.y0 === bBox.y0 && aBox.x0 === bBox.x0) {
return 0;
}
if (isATopmost && isALeftmost && !(isBTopmost && isBLeftmost)) {
return -1;
}
if (isBTopmost && isBLeftmost && !(isATopmost && isALeftmost)) {
return 1;
}
if (isATopmost && isBLeftmost) {
if (aBox.y1 < bBox.y0) {
return -1;
}
else if (bBox.x1 < aBox.x0) {
return 1;
}
else {
return aBox.y0 - bBox.y0;
}
}
if (isBTopmost && isALeftmost) {
if (bBox.y1 < aBox.y0) {
return 1;
}
else if (aBox.x1 < bBox.x0) {
return -1;
}
else {
return aBox.y0 - bBox.y0;
}
}
if (aBox.y0 === bBox.y0) {
return aBox.x0 - bBox.x0;
}
if (aBox.x0 === bBox.x0) {
return aBox.y0 - bBox.y0;
}
const yDiff = aBox.y0 - bBox.y0;
if (Math.abs(yDiff) > 10) {
return yDiff;
} else {
return aBox.x0 - bBox.x0;
}
});
contourTreeNode.children.forEach(sortChildrenByDirection);
}
function determineAlignment(node) {
let children = node.children;
let primaryNode = children[0];
if (primaryNode) primaryNode.layoutDirection = "column";
for (let i = 1; i < children.length; ++i) {
if (children[i].bbox.x0 >= primaryNode.bbox.x1) {
children[i].layoutDirection = "row";
} else {
children[i].layoutDirection = "column";
}
primaryNode = children[i];
}
node.children.forEach(determineAlignment);
}
function generateCode(node, prevTop = 0, prevLeft = 0) {
if (node.children.length < 1) return "";
const children = node.children;
let columns = ``;
let maxPrevHeight = children[0].bbox.y1;
let row = `${wrapper(
children[0],
generateCode(children[0], children[0].bbox.y1, children[0].bbox.x1),
"element",
prevLeft,
prevTop
)}`;
for (let i = 1; i < children.length; ++i) {
if (children[i].layoutDirection === "row") {
maxPrevHeight = Math.max(maxPrevHeight, children[i].bbox.y1);
row += wrapper(
children[i],
generateCode(children[i], children[i].bbox.y1, children[i].bbox.x1),
"element",
children[i - 1].bbox.x1,
children[0].bbox.y0
);
} else {
let rowEnd = wrapper(row, null, "flex", null, null);
columns += rowEnd;
row = `${wrapper(
children[i],
generateCode(children[i], children[i].bbox.y1, children[i].bbox.x1),
"element",
prevLeft,
maxPrevHeight
)}`;
}
}
if (row) {
columns += wrapper(row, null, "flex", null, null);
}
return wrapper(columns, null, "cover", null, null);
}
function wrapper(parent, child, type, prevX, topY) {
if (type == "cover") {
return `<div>${parent}</div>n`;
} else if (type === "flex") {
return `<div style="display: flex">${parent}</div>n`;
} else if (type === "element") {
return `<${parent.element} style="width: ${
parent.bbox.x1 - parent.bbox.x0
}px; height: ${parent.bbox.y1 - parent.bbox.y0}px; background-color: ${
parent.bgColor
? `rgba(
${parent.bgColor.r},
${parent.bgColor.g},
${parent.bgColor.b},
${parent.bgColor.a}
)`
: `rgba(0, 0, 0, 0)`
}; color: ${
parent.color
? `rgba(${parent.color.r}, ${parent.color.g}, ${parent.color.b}, ${parent.color.a})`
: `rgba(0, 0, 0, 0)`
}; margin-left: ${parent.bbox.x0 - prevX}px; margin-top: ${
parent.bbox.y0 - topY
}px;">${child || parent.text || ''}</${parent.element}>
`;
}
}
First of all the arrangement method I came up with was a failure and didn’t work on many cases. Even if it was successfull, the above generateCode function I came up with is not gonna work as intended as it will result in wrong margin arrangement. Sorry for being an amateur at this
I just want a way I can make arrange the data as a proper html which is about 70-80% accurate and easily configurable. There has to be a solution. You know those drag-and-drop page builder’s (like wix). They make a proper design from drag-and-drop creations. They must be using position data of each components a user places and then somehow makes a working page out of it.