Currently, js sends the video to Python opencv through the client cam, recognizes the nails, and synthesizes the nail art image on the nails. Afterwards, the processed video is sent back to js by websocket to show a video with nails on it, and the post-processed video is gradually slowing down. May I know what the problem is?
python opencv code
@app.get("/client")
async def client(request: Request):
# /templates/client.html파일을 response함
print("request : " ,request)
return templates.TemplateResponse("client.html", {"request":request})
# 웹소켓 설정 ws://127.0.0.1:8000/ws 로 접속할 수 있음
@app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
# print(f"client connected : {websocket.client}")
await websocket.accept() # client의 websocket접속 허용
await websocket.send_text(f"Welcome client : {websocket.client}")
i = 0
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
# print(f'{mp_hands} 일단 선언?')
args = {
"model": "./model/export_model_008/frozen_inference_graph.pb",
# "model":"/media/todd/38714CA0C89E958E/147/yl_tmp/readingbook/model/export_model_015/frozen_inference_graph.pb",
"labels": "./record/classes.pbtxt",
# "labels":"record/classes.pbtxt" ,
"num_classes": 1,
"min_confidence": 0.6,
"class_model": "../model/class_model/p_class_model_1552620432_.h5"}
# print(args)
COLORS = np.random.uniform(0, 255, size=(args["num_classes"], 3))
# print(COLORS)
print('1 : 모델 생성')
model = tf.Graph()
with model.as_default():
print("> ====== loading NAIL frozen graph into memory")
graphDef = tf.GraphDef()
with tf.gfile.GFile(args["model"], "rb") as f:
serializedGraph = f.read()
graphDef.ParseFromString(serializedGraph)
tf.import_graph_def(graphDef, name="")
# sess = tf.Session(graph=graphDef)
print("> ====== NAIL Inference graph loaded.")
# return graphDef, sess
with mp_hands.Hands(min_detection_confidence=0.8,min_tracking_confidence=0.5) as hands:
with model.as_default():
with tf.Session(graph=model) as sess:
print('2 : 텐서플로우 실행')
imageTensor = model.get_tensor_by_name("image_tensor:0")
boxesTensor = model.get_tensor_by_name("detection_boxes:0")
scoresTensor = model.get_tensor_by_name("detection_scores:0")
classesTensor = model.get_tensor_by_name("detection_classes:0")
numDetections = model.get_tensor_by_name("num_detections:0")
drawboxes = []
while True:
print('3 : 캠읽음')
data = await websocket.receive_text() # client 메시지 수신대기
data = data.split(',')[1]
imgtest = imread(io.BytesIO(base64.b64decode(data)))
# print(imgtest)
img = cv2.imread("sss.png")
frame = cv2.flip(imgtest, 1)
image = frame
(H, W) = image.shape[:2]
# print("H,W:", (H, W))
output = image.copy()
# output[0:450, 0:450] = img
img_ff, bin_mask, res = ff.find_hand_old(image.copy())
image = cv2.cvtColor(res, cv2.COLOR_BGR2RGB)
image_2 = cv2.cvtColor(output, cv2.COLOR_BGR2RGB)
results = hands.process(image_2)
image = np.expand_dims(image, axis=0)
image_2 = cv2.cvtColor(image_2, cv2.COLOR_RGB2BGR)
imageHeight, imageWidth, _ = image_2.shape
# print('여기까진 옴')
if results.multi_hand_landmarks:
for num, hand in enumerate(results.multi_hand_landmarks):
normalizedLandmark = hand.landmark[12]
normalizedLandmark_2 = hand.landmark[11]
pixelCoordinatesLandmark = mp_drawing._normalized_to_pixel_coordinates(normalizedLandmark.x, normalizedLandmark.y, imageWidth, imageHeight)
pixelCoordinatesLandmark_2 = mp_drawing._normalized_to_pixel_coordinates(normalizedLandmark_2.x, normalizedLandmark_2.y, imageWidth, imageHeight)
tanTheta = ((pixelCoordinatesLandmark_2[0]-pixelCoordinatesLandmark[0]))/((pixelCoordinatesLandmark_2[1]-pixelCoordinatesLandmark[1]))
theta = np.arctan(tanTheta)
try:
angle = theta*180/math.pi
except:
print('수평')
# cv2.imshow('Output',image_2)
(boxes, scores, labels, N) = sess.run(
[boxesTensor, scoresTensor, classesTensor, numDetections],
feed_dict={imageTensor: image})
boxes = np.squeeze(boxes)
scores = np.squeeze(scores)
labels = np.squeeze(labels)
boxnum = 0
box_mid = (0, 0)
for (box, score, label) in zip(boxes, scores, labels):
if score < args["min_confidence"]:
continue
# scale the bounding box from the range [0, 1] to [W, H]
boxnum = boxnum + 1
(startY, startX, endY, endX) = box
startX = int(startX * W)
startY = int(startY * H)
endX = int(endX * W)
endY = int(endY * H)
# print(startX,startY,endX,endY)
X_mid = startX + int(abs(endX - startX) / 2)
Y_mid = startY + int(abs(endY - startY) / 2)
box_mid = (X_mid, Y_mid)
# draw the prediction on the output image
label_name = 'nail'
# idx = int(label["id"]) - 1
idx = 0
# x = 가로 y = 세로 점
# print(startX,startY,endX,endY)
startX = startX -20
startY = startY -20
endX = endX +20
endY = endY +20
wi = int(abs(endY-startY))
he = int(abs(endX-startX))
cat_sticker = cv2.resize(img,(wi,he))
# print(cat_sticker.shape)
# 회전
img_rotate = ff.rotate_image(cat_sticker,angle)
# print('회전된 이미지 shape : ' ,img_rotate.shape)
xx = X_mid -wi // 2
yy = Y_mid - he // 2
if xx < 0:
cat_sticer = cat_sticker[:,xx:]
if yy < 0:
cat_sticker = cat_sticker[-yy:,:]
try :
sticker_area = image_2[yy:yy+img_rotate.shape[0],xx:xx+img_rotate.shape[1]]
image_2[yy:yy+img_rotate.shape[0], xx:xx+img_rotate.shape[1]] = np.where(img_rotate==0,sticker_area,img_rotate).astype(np.uint8)
except :
print('에러')
encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 65]
buffer = cv2.imencode('.jpg', image_2,encode_param)[1]
image_2 = buffer.tobytes()
await websocket.send_text(buffer.tobytes())
client code
<html>
<head>
<title>Hello</title>
</head>
<script async src="https://docs.opencv.org/3.4/opencv.js"></script>
<!--script async src="./opencv.js"></script-->
<body>
<video id="videoInput" style="display:none"></video>
<canvas id="videoOutput"></canvas>
<button onclick=stream()>Send</button>
<canvas id="testt" width="300" height="200" style="border: 1px solid red"></canvas>
<div id="status">
Connection failed. Somebody may be using the socket.
</div>
<video id=""video></video>
</body>
<script>
var w = 640, h = 480;
var url = "ws://localhost:8000/ws"
var ws = new WebSocket(url);
let testt = document.getElementById("testt");
testt.width = w;
testt.height = h;
ws.addEventListener('open', (e) => {
// console.log('ㅇㅇㅇㅇㄹㄴㅇㅁㄹㄴㅁ래제머ㅏ레ㅐㅈ머')
document.getElementById("status").innerHTML = "Opened";
});
ws.addEventListener('message', (e) => {
console.log(e.data)
let ctx = testt.getContext("2d");
let image = new Image();
URL.revokeObjectURL( oldObjectUrl );
try {
image.src = URL.createObjectURL(e.data);
}
catch {
// console.log('여기로?')
image.srcObject = e.data
}
console.log(image)
image.addEventListener("load", (e) => {
// console.log('들어옴ㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁㅁ')
ctx.drawImage(image, 0, 0, testt.width, testt.height);
});
});
ws.onopen = function(){
console.log("Websocket is connected.");
}
navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
var constraints = {audio: false, video: true};
var video = document.getElementById("videoInput");
video.width = w;
video.height = h;
function successCallback(stream){
console.log('엥들어옴')
video.srcObject = stream;
video.play();
}
function errorCallback(error){
console.log(error);
}
navigator.getUserMedia(constraints, successCallback, errorCallback);
var canvas = document.getElementById("videoOutput");
canvas.width = w;
canvas.height = h;
var ctx = canvas.getContext("2d");
function processImage(){
ctx.drawImage(video, 0, 0, w, h);
setTimeout(processImage, 1);
}
processImage();
function stream(){
setInterval(sendImage, 30);
}
function sendImage(){
var rawData = canvas.toDataURL("image/jpeg", 3);
// console.log(rawData)
ws.send(rawData);
}
</script>
</html>