three.js 视频融合

MixVideo.js代码：

//视频融合

import * as THREE from '../build/three.module.js';
import { API } from '../js.my/API.js';
import { Msg } from '../js.my/Msg.js';
import { createDebounce } from '../js.my/Utils.js';
import { guiParams, createGuiParams } from '../js.my/MixVideoGui.js'

let api = new API();
let msg = new Msg();

let mesh;
let material;
let videoTexture;
let loadingVideoTexture;

let debounce = createDebounce(2000);

function createGeometry(params, mixVideoBounds) {
    let geometry = new THREE.Geometry();
    if (!params) {
        geometry.vertices.push(new THREE.Vector3(mixVideoBounds[0].x, mixVideoBounds[0].y, mixVideoBounds[0].z));
        geometry.vertices.push(new THREE.Vector3(mixVideoBounds[1].x, mixVideoBounds[1].y, mixVideoBounds[1].z));
        geometry.vertices.push(new THREE.Vector3(mixVideoBounds[2].x, mixVideoBounds[2].y, mixVideoBounds[2].z));
        geometry.vertices.push(new THREE.Vector3(mixVideoBounds[3].x, mixVideoBounds[3].y, mixVideoBounds[3].z));
    } else {
        geometry.vertices.push(new THREE.Vector3(params.bounds0_x, params.bounds0_y, params.bounds0_z));
        geometry.vertices.push(new THREE.Vector3(params.bounds1_x, params.bounds1_y, params.bounds1_z));
        geometry.vertices.push(new THREE.Vector3(params.bounds2_x, params.bounds2_y, params.bounds2_z));
        geometry.vertices.push(new THREE.Vector3(params.bounds3_x, params.bounds3_y, params.bounds3_z));
    }

    let normal = new THREE.Vector3(0, 0, 1);

    let face0 = new THREE.Face3(0, 1, 2, normal);
    let face1 = new THREE.Face3(0, 2, 3, normal);
    geometry.faces.push(face0, face1);

    let t0 = new THREE.Vector2(0, 0);
    let t1 = new THREE.Vector2(1, 0);
    let t2 = new THREE.Vector2(1, 1);
    let t3 = new THREE.Vector2(0, 1);
    let uv1 = [t0, t1, t2];
    let uv2 = [t0, t2, t3];
    geometry.faceVertexUvs[0].push(uv1, uv2);

    geometry.computeFaceNormals();
    geometry.computeVertexNormals();

    return geometry;
}

let changeMaterialMap = () => {
    if (material && videoTexture && material.map === loadingVideoTexture) {
        material.map = videoTexture;
    }
};

function createVideoMesh(scene, fly, video, loadingVideo, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge) {

    videoTexture = new THREE.VideoTexture(video);
    videoTexture.minFilter = THREE.LinearFilter;
    videoTexture.magFilter = THREE.LinearFilter;
    videoTexture.format = THREE.RGBFormat;

    loadingVideoTexture = new THREE.VideoTexture(loadingVideo);
    loadingVideoTexture.minFilter = THREE.LinearFilter;
    loadingVideoTexture.magFilter = THREE.LinearFilter;
    loadingVideoTexture.format = THREE.RGBFormat;

    material = new THREE.MeshBasicMaterial({
        map: loadingVideoTexture,
        color: 0xffffff,
        depthTest: false,
        transparent: true,
        opacity: 0.95
    });

    mesh = new THREE.Mesh(createGeometry(undefined, mixVideoBounds), material);
    scene.add(mesh);

    fly.moveCameraOnly(mixVideoCameraPosition, mixVideoCameraTarge);

    createGuiParams(mixVideoBounds, () => {
        mesh.geometry = createGeometry(guiParams);

        let mixVideoBounds = [
            { x: guiParams.bounds0_x, y: guiParams.bounds0_y, z: guiParams.bounds0_z },
            { x: guiParams.bounds1_x, y: guiParams.bounds1_y, z: guiParams.bounds1_z },
            { x: guiParams.bounds2_x, y: guiParams.bounds2_y, z: guiParams.bounds2_z },
            { x: guiParams.bounds3_x, y: guiParams.bounds3_y, z: guiParams.bounds3_z }
        ];

        for (let i = 0; i < mixVideoBounds.length - 1; i++) {
            mixVideoBounds[i].x = parseFloat(mixVideoBounds[i].x.toFixed(6));
            mixVideoBounds[i].y = parseFloat(mixVideoBounds[i].y.toFixed(6));
            mixVideoBounds[i].z = parseFloat(mixVideoBounds[i].z.toFixed(6));
        }

        let data = {
            id: cameraId,
            mix_video_bounds: JSON.stringify(mixVideoBounds),
        }

        debounce(() => {
            api.updatePtCameraInfo(data, () => {
                msg.show("视频融合参数已保存");
            });
        });
    });
}

function mixVideo(scene, fly, cameraIndexCode, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge) {
    msg.show("即将加载视频请稍等");
    mesh && scene.remove(mesh);

    //创建DOM
    if ($('#mixVideo').length == 0) {

        //video标签，外层div测试用
        let videoStr = `
            <div id="mixVideoDiv" style="display:none; z-index: -999999; position: absolute; float: left; top: 0; left: 0; background-color: #ff0000;">
                <video id="mixVideo" style="100px; height:100px;" loop="loop" poster="images/mix-video/loading.gif">
                    <source src="../../video/videoPlane.mp4" type="video/mp4">
                </video>
                <video id="loadingVideo" style="100px; height:100px;" loop="loop" >
                    <source src="images/mix-video/loading.mp4" type="video/mp4">
                </video>  
            </div>`

        $('body').append(videoStr);
    }

    let video = document.getElementById('mixVideo');
    let loadingVideo = document.getElementById('loadingVideo');

    //取流
    // api.getVideoUrl(cameraIndexCode, data => {
    //     createVideoMesh(scene, fly, video, loadingVideo, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge);
    //     hlsPlay(video, loadingVideo, data);
    // }, errMsg => {
    //     playTestMp4(scene, fly, video, loadingVideo, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge);

    //     msg.show("取流失败：" + errMsg);
    // });

    //测试播放hls流
    let testUrl = 'http://playertest.longtailvideo.com/adaptive/bipbop/gear4/prog_index.m3u8';
    let testUrl2 = 'https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8';
    createVideoMesh(scene, fly, video, loadingVideo, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge);
    hlsPlay(video, loadingVideo, testUrl);

}

/** 播放hls流 */
function hlsPlay(video, loadingVideo, url) {
    loadingVideo.play();

    if (Hls.isSupported()) {
        const hls = new Hls();
        hls.loadSource(url);
        hls.attachMedia(video);
        hls.on(Hls.Events.MEDIA_ATTACHED, () => {

        });
        hls.on(Hls.Events.MANIFEST_PARSED, () => {
            video.play();
        });
        hls.on(Hls.Events.ERROR, (event, data) => {

        });
        hls.on(Hls.Events.FRAG_LOADED, () => {
            changeMaterialMap();
        });
    } else {
        msg.show("您的浏览器不支持播放该视频流");
    }
}

function playTestMp4(scene, fly, video, loadingVideo, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge) {
    loadingVideo.play();
    video.play();
    createVideoMesh(scene, fly, video, loadingVideo, cameraId, mixVideoBounds, mixVideoCameraPosition, mixVideoCameraTarge);
    changeMaterialMap();
}

export { mixVideo }

如何使用：调用mixVideo方法，把scene、fly(用于场景飞行)和其它配置的参数传给它即可

涉及到的变量说明：

video 视频标签DOM

loadingVideo 视频加载出来前的loading动画的DOM，mp4格式

mixVideoBounds 播放视频的Geometry的四个顶点的坐标

mixVideoCameraPosition 场景相机position(PerspectiveCamera对象的position)

mixVideoCameraTarge 场景相机target(OrbitControls.js的OrbitControls对象的target)

mixVideoBounds参数不好调，我做了一个调参的功能，当参数调整时，自动保存到数据库

MixVideoGui.js代码：

//控制视频融合播放范围

import { GUI } from "../js/libs/dat.gui.module.js";

let gui = new GUI({ autoPlace: false,  260, hideable: true });

GUI.TEXT_CLOSED = '隐藏';
GUI.TEXT_OPEN = '展开';

let guiParams;

let folderLeftBottom;
let folderRightBottom;
let folderRightTop;
let folderLeftTop;

function createGuiParams(mixVideoBounds, onChange) {
    if (folderLeftBottom) {
        gui.removeFolder(folderLeftBottom);
        gui.removeFolder(folderRightBottom);
        gui.removeFolder(folderRightTop);
        gui.removeFolder(folderLeftTop);
    }

    guiParams = new function () {
        this.bounds0_x = mixVideoBounds[0].x;
        this.bounds0_y = mixVideoBounds[0].y;
        this.bounds0_z = mixVideoBounds[0].z;

        this.bounds1_x = mixVideoBounds[1].x;
        this.bounds1_y = mixVideoBounds[1].y;
        this.bounds1_z = mixVideoBounds[1].z;

        this.bounds2_x = mixVideoBounds[2].x;
        this.bounds2_y = mixVideoBounds[2].y;
        this.bounds2_z = mixVideoBounds[2].z;

        this.bounds3_x = mixVideoBounds[3].x;
        this.bounds3_y = mixVideoBounds[3].y;
        this.bounds3_z = mixVideoBounds[3].z;
    }

    folderLeftBottom = gui.addFolder('左下');
    folderRightBottom = gui.addFolder('右下');
    folderRightTop = gui.addFolder('右上');
    folderLeftTop = gui.addFolder('左上');

    folderLeftBottom.open();
    folderRightBottom.open();
    folderRightTop.open();
    folderLeftTop.open();

    let guiParamsDelta = 1000;
    let guiParamsDeltaY = 1000;
    let step = 0.1;

    let paramCtrls = [
        folderLeftBottom.add(guiParams, "bounds0_x", guiParams.bounds0_x - guiParamsDelta, guiParams.bounds0_x + guiParamsDelta, step),
        folderLeftBottom.add(guiParams, "bounds0_y", guiParams.bounds0_y - guiParamsDeltaY, guiParams.bounds0_y + guiParamsDeltaY, step),
        folderLeftBottom.add(guiParams, "bounds0_z", guiParams.bounds0_z - guiParamsDelta, guiParams.bounds0_z + guiParamsDelta, step),

        folderRightBottom.add(guiParams, "bounds1_x", guiParams.bounds1_x - guiParamsDelta, guiParams.bounds1_x + guiParamsDelta, step),
        folderRightBottom.add(guiParams, "bounds1_y", guiParams.bounds1_y - guiParamsDeltaY, guiParams.bounds1_y + guiParamsDeltaY, step),
        folderRightBottom.add(guiParams, "bounds1_z", guiParams.bounds1_z - guiParamsDelta, guiParams.bounds1_z + guiParamsDelta),

        folderRightTop.add(guiParams, "bounds2_x", guiParams.bounds2_x - guiParamsDelta, guiParams.bounds2_x + guiParamsDelta, step),
        folderRightTop.add(guiParams, "bounds2_y", guiParams.bounds2_y - guiParamsDeltaY, guiParams.bounds2_y + guiParamsDeltaY, step),
        folderRightTop.add(guiParams, "bounds2_z", guiParams.bounds2_z - guiParamsDelta, guiParams.bounds2_z + guiParamsDelta, step),

        folderLeftTop.add(guiParams, "bounds3_x", guiParams.bounds3_x - guiParamsDelta, guiParams.bounds3_x + guiParamsDelta, step),
        folderLeftTop.add(guiParams, "bounds3_y", guiParams.bounds3_y - guiParamsDeltaY, guiParams.bounds3_y + guiParamsDeltaY, step),
        folderLeftTop.add(guiParams, "bounds3_z", guiParams.bounds3_z - guiParamsDelta, guiParams.bounds3_z + guiParamsDelta, step)
    ];

    paramCtrls.forEach(ctrl => ctrl.onChange(onChange));

    if ($('#guiDomElement').length == 0) {
        let guiDomElement = `<div id="guiDomElement" style="position:absolute; z-index:1990; float:left; left:165px; top:220px; 260px;" ></div> `;
        $('body').append(guiDomElement);
        $('#guiDomElement').append(gui.domElement);
        gui.open();
    }
}

export { gui, guiParams, createGuiParams }

效果图：

没有真实的视频，随便找了个在线的hls流

效果图gif：

说明：第1个模拟的是平视的摄像机，第2个和第3个模拟的是俯视的摄像机，第4个没有配置视频融合相关参数，直接弹出视频播放对话框。

 

 mixVideoBounds参数调整效果图：

现场测试效果图： 

效果不怎么样，也可能只是参数没调好。