[kaka/cakelight.git] / src / kaka / cakelight / VideoFrame.java

package kaka.cakelight;

import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;

import static kaka.cakelight.Main.saveFile;
import static kaka.cakelight.Main.timeIt;

public class VideoFrame {
    private byte[] data;
    private Configuration config;
//    private Mat colImage;
//    private Mat rowImage;
    private Mat converted;
    private Mat[] images;

    private VideoFrame(byte[] data) {
        this.data = data;
    }

    public static VideoFrame of(byte[] data, Configuration config) {
        VideoFrame frame = new VideoFrame(data);
        frame.config = config;
        if (config.video.mjpg) {
            frame.convertJpg();
        } else {
            frame.convert();
        }
        return frame;
    }

    private void convertJpg() {
        Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC3); // 8-bit, unsigned, 3 channels
        src.put(0, 0, data);
//        save(src, "/home/kaka/test-src.data");

        converted = new Mat();
        Imgproc.cvtColor(src, converted, Imgproc.COLOR_BGR2RGB);

        Core.flip(converted, converted, 0); // up-side down
//        save(converted, "/home/kaka/test-converted.data");
        int mysteriousPixelShift = 18;
        converted = converted.submat( // crop mysterious pixel shift
                0,
                converted.rows(),
                mysteriousPixelShift,
                converted.cols() - mysteriousPixelShift
        );
//        save(converted, "/home/kaka/test-croppedAgain.data");
        model4(converted, Imgproc.INTER_AREA);
        src.release();
        converted.release();
    }

    private void convert() {
        /* TODO: how to do this?
        1) Resize to an image with the size of the number of leds and use config to define how many pixels deep into the screen to use.
        2) Resize to 16x9 and use 2 pixels of depth (or maybe 3) and interpolate for each led.
        3) Resize to 2 images where each led uses 2 pixels:
            vertical   - 16 x <#leds>
            horizontal - <#leds> x 9
        4) Resize to cols x rows first, then resize to a vertical and a horizontal like in (3).
         */
        Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC2); // 8-bit, unsigned, 2 channels
        src.put(0, 0, data);

//        Mat converted = new Mat();
//        Mat resized = new Mat();
//
//        timeIt("total", () -> {
//            timeIt("yuyv2rgb", () -> Imgproc.cvtColor(src, converted, Imgproc.COLOR_YUV2RGB_YUYV)); // 3.5 - 4.0 ms
//            timeIt("resizing", () -> Imgproc.resize(converted, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, Imgproc.INTER_AREA)); // INTER_AREA is the best for shrinking, but also the slowest (~1.5 ms)
//        });

        Mat cropped = src.submat(
                config.video.crop.top,
                config.video.height - config.video.crop.bottom,
                config.video.crop.left,
                config.video.width - config.video.crop.right
        );
        converted = new Mat();
        Imgproc.cvtColor(cropped, converted, config.video.format);
//        timeIt("model 1", () -> model1(converted, Imgproc.INTER_AREA));
//        timeIt("model 2", () -> model2(converted, Imgproc.INTER_AREA));
//        timeIt("model 3", () -> model3(converted, Imgproc.INTER_AREA));
        timeIt("model 4", () -> model4(converted, Imgproc.INTER_AREA));
//        save(converted, "/home/kaka/test-converted.data");
//        save(resized, "/home/kaka/test-resized.data");
        src.release();
        cropped.release();
    }

    private void model1(Mat src, int interpolation) {
        Mat resized = new Mat();
        Imgproc.resize(src, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, interpolation);
    }

    private void model2(Mat src, int interpolation) {
        Mat resized = new Mat();
        Imgproc.resize(src, resized, new Size(16, 9), 0, 0, interpolation);
    }

    private void model3(Mat src, int interpolation) {
//        colImage = new Mat();
//        rowImage = new Mat();
//        Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
//        Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
    }

    private void model4(Mat src, int interpolation) {
        int width = 3 * src.cols() / 16;
        int height = 3 * src.rows() / 9;
        Mat[] cropped = new Mat[] {
                /* LEFT   */ src.submat(0, src.rows(), 0, width),
                /* RIGHT  */ src.submat(0, src.rows(), src.cols() - width, src.cols()),
                /* TOP    */ src.submat(0, height, 0, src.cols()),
                /* BOTTOM */ src.submat(src.rows() - height, src.rows(), 0, src.cols()),
        };
        images = new Mat[] {new Mat(), new Mat(), new Mat(), new Mat()};
//        Imgproc.resize(cropped[ListPosition.LEFT.ordinal()], images[ListPosition.LEFT.ordinal()], new Size(3, config.leds.rows), 0, 0, interpolation);
        Imgproc.resize(cropped[0], images[0], new Size(3, config.leds.rows), 0, 0, interpolation);
        Imgproc.resize(cropped[1], images[1], new Size(3, config.leds.rows), 0, 0, interpolation);
        Imgproc.resize(cropped[2], images[2], new Size(config.leds.cols, 3), 0, 0, interpolation);
        Imgproc.resize(cropped[3], images[3], new Size(config.leds.cols, 3), 0, 0, interpolation);
//        Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
//        Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
    }

    private Color wrappedGetLedColor(ListPosition listPosition, int xy) {
        Color c = getLedColor(listPosition, xy);
        double[] hsv = c.toHSV();
        double saturation = config.video.saturation >= 0.5
                ? hsv[1] + (config.video.saturation - 0.5) * 2 * (1 - hsv[1])
                : hsv[1] - (1 - config.video.saturation * 2) * hsv[1];
        return Color.hsv(hsv[0], saturation, hsv[2]);
    }

    private Color getLedColor(ListPosition listPosition, int xy) {
        // TODO: maybe use highest value from pixels? 100 % from 1st, 66 % from 2nd, 33 % from 3rd. colors might be strange.
        switch (listPosition) {
            case LEFT:
                return interpolatedRowColor(images[0], xy, 0, 1, 2);
//                return interpolatedRowColor(xy, 0, 1, 2);
            case RIGHT:
                return interpolatedRowColor(images[1], xy, 2, 1, 0);
//                return interpolatedRowColor(xy, 15, 14, 13);
            case TOP:
                return interpolatedColColor(images[2], xy, 0, 1, 2);
//                return interpolatedColColor(xy, 0, 1, 2);
            case BOTTOM:
                return interpolatedColColor(images[3], xy, 2, 1, 0);
//                return interpolatedColColor(xy, 8, 7, 6);
        }
        return null;
    }

    //    private Color interpolatedRowColor(int y, int x1, int x2, int x3) {
    private Color interpolatedRowColor(Mat rowImage, int y, int x1, int x2, int x3) {
        return pixelToColor(rowImage, x3, y).interpolate(pixelToColor(rowImage, x2, y), 0.65).interpolate(pixelToColor(rowImage, x1, y), 0.65);
    }

//    private Color interpolatedColColor(int x, int y1, int y2, int y3) {
    private Color interpolatedColColor(Mat colImage, int x, int y1, int y2, int y3) {
        return pixelToColor(colImage, x, y3).interpolate(pixelToColor(colImage, x, y2), 0.65).interpolate(pixelToColor(colImage, x, y1), 0.65);
    }

    private Color pixelToColor(Mat image, int x, int y) {
        byte[] rgb = new byte[3];
        image.get(y, x, rgb);
        return Color.rgb(rgb[0] & 0xff, rgb[1] & 0xff, rgb[2] & 0xff);
    }

    private void save(Mat mat, String filepath) {
        byte[] data = new byte[mat.cols() * mat.rows() * mat.channels()];
        mat.get(0, 0, data);
        saveFile(data, filepath);
    }

    public byte[] getData() {
        byte[] buff = new byte[(int) (converted.total() * converted.channels())];
        converted.get(0, 0, buff);
        return buff;
    }

//    public Mat getColImage() {
//        return colImage;
//    }

//    public Mat getRowImage() {
//        return rowImage;
//    }

    public Mat getConvertedImage() {
        return converted;
    }

    /**
     * Creates a LED frame going counter-clockwise from the bottom-left corner, sans the corners.
     */
    public LedFrame getLedFrame() {
        LedFrame frame = LedFrame.from(config);
        int led = 0;

        if (config.video.list.bottom)
            for (int i = 0; i < config.leds.cols; i++)      frame.setLedColor(led++, wrappedGetLedColor(ListPosition.BOTTOM, i));
	else
	    for (int i = 0; i < config.leds.cols; i++)      frame.setLedColor(led++, Color.BLACK);

        if (config.video.list.right)
            for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.RIGHT, i));
        else
            for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);

        if (config.video.list.top)
            for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.TOP, i));
        else
            for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);

        if (config.video.list.left)
            for (int i = 0; i < config.leds.rows; i++)      frame.setLedColor(led++, wrappedGetLedColor(ListPosition.LEFT, i));
        else
            for (int i = 0; i < config.leds.rows; i++)      frame.setLedColor(led++, Color.BLACK);

        return frame;
    }

    private enum ListPosition {
        LEFT,
        RIGHT,
        TOP,
        BOTTOM
    }
}
Commit	Line	Data
	1	package kaka.cakelight;
	2
	3	import org.opencv.core.Core;
	4	import org.opencv.core.CvType;
	5	import org.opencv.core.Mat;
	6	import org.opencv.core.Size;
	7	import org.opencv.imgproc.Imgproc;
	8
	9	import static kaka.cakelight.Main.saveFile;
	10	import static kaka.cakelight.Main.timeIt;
	11
	12	public class VideoFrame {
	13	private byte[] data;
	14	private Configuration config;
	15	// private Mat colImage;
	16	// private Mat rowImage;
	17	private Mat converted;
	18	private Mat[] images;
	19
	20	private VideoFrame(byte[] data) {
	21	this.data = data;
	22	}
	23
	24	public static VideoFrame of(byte[] data, Configuration config) {
	25	VideoFrame frame = new VideoFrame(data);
	26	frame.config = config;
	27	if (config.video.mjpg) {
	28	frame.convertJpg();
	29	} else {
	30	frame.convert();
	31	}
	32	return frame;
	33	}
	34
	35	private void convertJpg() {
	36	Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC3); // 8-bit, unsigned, 3 channels
	37	src.put(0, 0, data);
	38	// save(src, "/home/kaka/test-src.data");
	39
	40	converted = new Mat();
	41	Imgproc.cvtColor(src, converted, Imgproc.COLOR_BGR2RGB);
	42
	43	Core.flip(converted, converted, 0); // up-side down
	44	// save(converted, "/home/kaka/test-converted.data");
	45	int mysteriousPixelShift = 18;
	46	converted = converted.submat( // crop mysterious pixel shift
	47	0,
	48	converted.rows(),
	49	mysteriousPixelShift,
	50	converted.cols() - mysteriousPixelShift
	51	);
	52	// save(converted, "/home/kaka/test-croppedAgain.data");
	53	model4(converted, Imgproc.INTER_AREA);
	54	src.release();
	55	converted.release();
	56	}
	57
	58	private void convert() {
	59	/* TODO: how to do this?
	60	1) Resize to an image with the size of the number of leds and use config to define how many pixels deep into the screen to use.
	61	2) Resize to 16x9 and use 2 pixels of depth (or maybe 3) and interpolate for each led.
	62	3) Resize to 2 images where each led uses 2 pixels:
	63	vertical - 16 x <#leds>
	64	horizontal - <#leds> x 9
	65	4) Resize to cols x rows first, then resize to a vertical and a horizontal like in (3).
	66	*/
	67	Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC2); // 8-bit, unsigned, 2 channels
	68	src.put(0, 0, data);
	69
	70	// Mat converted = new Mat();
	71	// Mat resized = new Mat();
	72	//
	73	// timeIt("total", () -> {
	74	// timeIt("yuyv2rgb", () -> Imgproc.cvtColor(src, converted, Imgproc.COLOR_YUV2RGB_YUYV)); // 3.5 - 4.0 ms
	75	// timeIt("resizing", () -> Imgproc.resize(converted, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, Imgproc.INTER_AREA)); // INTER_AREA is the best for shrinking, but also the slowest (~1.5 ms)
	76	// });
	77
	78	Mat cropped = src.submat(
	79	config.video.crop.top,
	80	config.video.height - config.video.crop.bottom,
	81	config.video.crop.left,
	82	config.video.width - config.video.crop.right
	83	);
	84	converted = new Mat();
	85	Imgproc.cvtColor(cropped, converted, config.video.format);
	86	// timeIt("model 1", () -> model1(converted, Imgproc.INTER_AREA));
	87	// timeIt("model 2", () -> model2(converted, Imgproc.INTER_AREA));
	88	// timeIt("model 3", () -> model3(converted, Imgproc.INTER_AREA));
	89	timeIt("model 4", () -> model4(converted, Imgproc.INTER_AREA));
	90	// save(converted, "/home/kaka/test-converted.data");
	91	// save(resized, "/home/kaka/test-resized.data");
	92	src.release();
	93	cropped.release();
	94	}
	95
	96	private void model1(Mat src, int interpolation) {
	97	Mat resized = new Mat();
	98	Imgproc.resize(src, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, interpolation);
	99	}
	100
	101	private void model2(Mat src, int interpolation) {
	102	Mat resized = new Mat();
	103	Imgproc.resize(src, resized, new Size(16, 9), 0, 0, interpolation);
	104	}
	105
	106	private void model3(Mat src, int interpolation) {
	107	// colImage = new Mat();
	108	// rowImage = new Mat();
	109	// Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
	110	// Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
	111	}
	112
	113	private void model4(Mat src, int interpolation) {
	114	int width = 3 * src.cols() / 16;
	115	int height = 3 * src.rows() / 9;
	116	Mat[] cropped = new Mat[] {
	117	/* LEFT */ src.submat(0, src.rows(), 0, width),
	118	/* RIGHT */ src.submat(0, src.rows(), src.cols() - width, src.cols()),
	119	/* TOP */ src.submat(0, height, 0, src.cols()),
	120	/* BOTTOM */ src.submat(src.rows() - height, src.rows(), 0, src.cols()),
	121	};
	122	images = new Mat[] {new Mat(), new Mat(), new Mat(), new Mat()};
	123	// Imgproc.resize(cropped[ListPosition.LEFT.ordinal()], images[ListPosition.LEFT.ordinal()], new Size(3, config.leds.rows), 0, 0, interpolation);
	124	Imgproc.resize(cropped[0], images[0], new Size(3, config.leds.rows), 0, 0, interpolation);
	125	Imgproc.resize(cropped[1], images[1], new Size(3, config.leds.rows), 0, 0, interpolation);
	126	Imgproc.resize(cropped[2], images[2], new Size(config.leds.cols, 3), 0, 0, interpolation);
	127	Imgproc.resize(cropped[3], images[3], new Size(config.leds.cols, 3), 0, 0, interpolation);
	128	// Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
	129	// Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
	130	}
	131
	132	private Color wrappedGetLedColor(ListPosition listPosition, int xy) {
	133	Color c = getLedColor(listPosition, xy);
	134	double[] hsv = c.toHSV();
	135	double saturation = config.video.saturation >= 0.5
	136	? hsv[1] + (config.video.saturation - 0.5) * 2 * (1 - hsv[1])
	137	: hsv[1] - (1 - config.video.saturation * 2) * hsv[1];
	138	return Color.hsv(hsv[0], saturation, hsv[2]);
	139	}
	140
	141	private Color getLedColor(ListPosition listPosition, int xy) {
	142	// TODO: maybe use highest value from pixels? 100 % from 1st, 66 % from 2nd, 33 % from 3rd. colors might be strange.
	143	switch (listPosition) {
	144	case LEFT:
	145	return interpolatedRowColor(images[0], xy, 0, 1, 2);
	146	// return interpolatedRowColor(xy, 0, 1, 2);
	147	case RIGHT:
	148	return interpolatedRowColor(images[1], xy, 2, 1, 0);
	149	// return interpolatedRowColor(xy, 15, 14, 13);
	150	case TOP:
	151	return interpolatedColColor(images[2], xy, 0, 1, 2);
	152	// return interpolatedColColor(xy, 0, 1, 2);
	153	case BOTTOM:
	154	return interpolatedColColor(images[3], xy, 2, 1, 0);
	155	// return interpolatedColColor(xy, 8, 7, 6);
	156	}
	157	return null;
	158	}
	159
	160	// private Color interpolatedRowColor(int y, int x1, int x2, int x3) {
	161	private Color interpolatedRowColor(Mat rowImage, int y, int x1, int x2, int x3) {
	162	return pixelToColor(rowImage, x3, y).interpolate(pixelToColor(rowImage, x2, y), 0.65).interpolate(pixelToColor(rowImage, x1, y), 0.65);
	163	}
	164
	165	// private Color interpolatedColColor(int x, int y1, int y2, int y3) {
	166	private Color interpolatedColColor(Mat colImage, int x, int y1, int y2, int y3) {
	167	return pixelToColor(colImage, x, y3).interpolate(pixelToColor(colImage, x, y2), 0.65).interpolate(pixelToColor(colImage, x, y1), 0.65);
	168	}
	169
	170	private Color pixelToColor(Mat image, int x, int y) {
	171	byte[] rgb = new byte[3];
	172	image.get(y, x, rgb);
	173	return Color.rgb(rgb[0] & 0xff, rgb[1] & 0xff, rgb[2] & 0xff);
	174	}
	175
	176	private void save(Mat mat, String filepath) {
	177	byte[] data = new byte[mat.cols() * mat.rows() * mat.channels()];
	178	mat.get(0, 0, data);
	179	saveFile(data, filepath);
	180	}
	181
	182	public byte[] getData() {
	183	byte[] buff = new byte[(int) (converted.total() * converted.channels())];
	184	converted.get(0, 0, buff);
	185	return buff;
	186	}
	187
	188	// public Mat getColImage() {
	189	// return colImage;
	190	// }
	191
	192	// public Mat getRowImage() {
	193	// return rowImage;
	194	// }
	195
	196	public Mat getConvertedImage() {
	197	return converted;
	198	}
	199
	200	/**
	201	* Creates a LED frame going counter-clockwise from the bottom-left corner, sans the corners.
	202	*/
	203	public LedFrame getLedFrame() {
	204	LedFrame frame = LedFrame.from(config);
	205	int led = 0;
	206
	207	if (config.video.list.bottom)
	208	for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.BOTTOM, i));
	209	else
	210	for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, Color.BLACK);
	211
	212	if (config.video.list.right)
	213	for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.RIGHT, i));
	214	else
	215	for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);
	216
	217	if (config.video.list.top)
	218	for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.TOP, i));
	219	else
	220	for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);
	221
	222	if (config.video.list.left)
	223	for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.LEFT, i));
	224	else
	225	for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, Color.BLACK);
	226
	227	return frame;
	228	}
	229
	230	private enum ListPosition {
	231	LEFT,
	232	RIGHT,
	233	TOP,
	234	BOTTOM
	235	}
	236	}