[kaka/cakelight.git] / src / kaka / cakelight / VideoFrame.java

package kaka.cakelight;

import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;

import static kaka.cakelight.Main.saveFile;
import static kaka.cakelight.Main.timeIt;

public class VideoFrame {
    private byte[] data;
    private Configuration config;
//    private Mat colImage;
//    private Mat rowImage;
    private Mat converted;
    private Mat[] images;

    private VideoFrame(byte[] data) {
        this.data = data;
    }

    public static VideoFrame of(byte[] data, Configuration config) {
        VideoFrame frame = new VideoFrame(data);
        frame.config = config;
        frame.convert();
        return frame;
    }

    private void convert() {
        /* TODO: how to do this?
        1) Resize to an image with the size of the number of leds and use config to define how many pixels deep into the screen to use.
        2) Resize to 16x9 and use 2 pixels of depth (or maybe 3) and interpolate for each led.
        3) Resize to 2 images where each led uses 2 pixels:
            vertical   - 16 x <#leds>
            horizontal - <#leds> x 9
        4) Resize to cols x rows first, then resize to a vertical and a horizontal like in (3).
         */
        Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC2); // 8-bit, unsigned, 2 channels
        src.put(0, 0, data);

//        Mat converted = new Mat();
//        Mat resized = new Mat();
//
//        timeIt("total", () -> {
//            timeIt("yuyv2rgb", () -> Imgproc.cvtColor(src, converted, Imgproc.COLOR_YUV2RGB_YUYV)); // 3.5 - 4.0 ms
//            timeIt("resizing", () -> Imgproc.resize(converted, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, Imgproc.INTER_AREA)); // INTER_AREA is the best for shrinking, but also the slowest (~1.5 ms)
//        });

        Mat cropped = src.submat(
                config.video.crop.top,
                config.video.height - config.video.crop.bottom,
                config.video.crop.left,
                config.video.width - config.video.crop.right
        );
        converted = new Mat();
        Imgproc.cvtColor(cropped, converted, config.video.format);
//        timeIt("model 1", () -> model1(converted, Imgproc.INTER_AREA));
//        timeIt("model 2", () -> model2(converted, Imgproc.INTER_AREA));
//        timeIt("model 3", () -> model3(converted, Imgproc.INTER_AREA));
        timeIt("model 4", () -> model4(converted, Imgproc.INTER_AREA));
//        save(converted, "/home/kaka/test-converted.data");
//        save(resized, "/home/kaka/test-resized.data");
        src.release();
        cropped.release();
    }

    private void model1(Mat src, int interpolation) {
        Mat resized = new Mat();
        Imgproc.resize(src, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, interpolation);
    }

    private void model2(Mat src, int interpolation) {
        Mat resized = new Mat();
        Imgproc.resize(src, resized, new Size(16, 9), 0, 0, interpolation);
    }

    private void model3(Mat src, int interpolation) {
//        colImage = new Mat();
//        rowImage = new Mat();
//        Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
//        Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
    }

    private void model4(Mat src, int interpolation) {
        int width = 3 * src.cols() / 16;
        int height = 3 * src.rows() / 9;
        Mat[] cropped = new Mat[] {
                /* LEFT   */ src.submat(0, src.rows(), 0, width),
                /* RIGHT  */ src.submat(0, src.rows(), src.cols() - width, src.cols()),
                /* TOP    */ src.submat(0, height, 0, src.cols()),
                /* BOTTOM */ src.submat(src.rows() - height, src.rows(), 0, src.cols()),
        };
        images = new Mat[] {new Mat(), new Mat(), new Mat(), new Mat()};
//        Imgproc.resize(cropped[ListPosition.LEFT.ordinal()], images[ListPosition.LEFT.ordinal()], new Size(3, config.leds.rows), 0, 0, interpolation);
        Imgproc.resize(cropped[0], images[0], new Size(3, config.leds.rows), 0, 0, interpolation);
        Imgproc.resize(cropped[1], images[1], new Size(3, config.leds.rows), 0, 0, interpolation);
        Imgproc.resize(cropped[2], images[2], new Size(config.leds.cols, 3), 0, 0, interpolation);
        Imgproc.resize(cropped[3], images[3], new Size(config.leds.cols, 3), 0, 0, interpolation);
//        Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
//        Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
    }

    private Color wrappedGetLedColor(ListPosition listPosition, int xy) {
        Color c = getLedColor(listPosition, xy);
        double[] hsv = c.toHSV();
        double saturation = config.video.saturation >= 0.5
                ? hsv[1] + (config.video.saturation - 0.5) * 2 * (1 - hsv[1])
                : hsv[1] - (1 - config.video.saturation * 2) * hsv[1];
        return Color.hsv(hsv[0], saturation, hsv[2]);
    }

    private Color getLedColor(ListPosition listPosition, int xy) {
        // TODO: maybe use highest value from pixels? 100 % from 1st, 66 % from 2nd, 33 % from 3rd. colors might be strange.
        switch (listPosition) {
            case LEFT:
                return interpolatedRowColor(images[0], xy, 0, 1, 2);
//                return interpolatedRowColor(xy, 0, 1, 2);
            case RIGHT:
                return interpolatedRowColor(images[1], xy, 2, 1, 0);
//                return interpolatedRowColor(xy, 15, 14, 13);
            case TOP:
                return interpolatedColColor(images[2], xy, 0, 1, 2);
//                return interpolatedColColor(xy, 0, 1, 2);
            case BOTTOM:
                return interpolatedColColor(images[3], xy, 2, 1, 0);
//                return interpolatedColColor(xy, 8, 7, 6);
        }
        return null;
    }

    //    private Color interpolatedRowColor(int y, int x1, int x2, int x3) {
    private Color interpolatedRowColor(Mat rowImage, int y, int x1, int x2, int x3) {
        return pixelToColor(rowImage, x3, y).interpolate(pixelToColor(rowImage, x2, y), 0.65).interpolate(pixelToColor(rowImage, x1, y), 0.65);
    }

//    private Color interpolatedColColor(int x, int y1, int y2, int y3) {
    private Color interpolatedColColor(Mat colImage, int x, int y1, int y2, int y3) {
        return pixelToColor(colImage, x, y3).interpolate(pixelToColor(colImage, x, y2), 0.65).interpolate(pixelToColor(colImage, x, y1), 0.65);
    }

    private Color pixelToColor(Mat image, int x, int y) {
        byte[] rgb = new byte[3];
        image.get(y, x, rgb);
        return Color.rgb(rgb[0] & 0xff, rgb[1] & 0xff, rgb[2] & 0xff);
    }

    private void save(Mat mat, String filepath) {
        byte[] data = new byte[mat.cols() * mat.rows() * mat.channels()];
        mat.get(0, 0, data);
        saveFile(data, filepath);
    }

    public byte[] getData() {
        byte[] buff = new byte[(int) (converted.total() * converted.channels())];
        converted.get(0, 0, buff);
        return buff;
    }

//    public Mat getColImage() {
//        return colImage;
//    }

//    public Mat getRowImage() {
//        return rowImage;
//    }

    public Mat getConvertedImage() {
        return converted;
    }

    /**
     * Creates a LED frame going counter-clockwise from the bottom-left corner, sans the corners.
     */
    public LedFrame getLedFrame() {
        LedFrame frame = LedFrame.from(config);
        int led = 0;

        if (config.video.list.bottom)
            for (int i = 0; i < config.leds.cols; i++)      frame.setLedColor(led++, wrappedGetLedColor(ListPosition.BOTTOM, i));
	else
	    for (int i = 0; i < config.leds.cols; i++)      frame.setLedColor(led++, Color.BLACK);

        if (config.video.list.right)
            for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.RIGHT, i));
        else
            for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);

        if (config.video.list.top)
            for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.TOP, i));
        else
            for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);

        if (config.video.list.left)
            for (int i = 0; i < config.leds.rows; i++)      frame.setLedColor(led++, wrappedGetLedColor(ListPosition.LEFT, i));
        else
            for (int i = 0; i < config.leds.rows; i++)      frame.setLedColor(led++, Color.BLACK);

        return frame;
    }

    private enum ListPosition {
        LEFT,
        RIGHT,
        TOP,
        BOTTOM
    }
}
Commit	Line	Data
	1	package kaka.cakelight;
	2
	3	import org.opencv.core.CvType;
	4	import org.opencv.core.Mat;
	5	import org.opencv.core.Size;
	6	import org.opencv.imgproc.Imgproc;
	7
	8	import static kaka.cakelight.Main.saveFile;
	9	import static kaka.cakelight.Main.timeIt;
	10
	11	public class VideoFrame {
	12	private byte[] data;
	13	private Configuration config;
	14	// private Mat colImage;
	15	// private Mat rowImage;
	16	private Mat converted;
	17	private Mat[] images;
	18
	19	private VideoFrame(byte[] data) {
	20	this.data = data;
	21	}
	22
	23	public static VideoFrame of(byte[] data, Configuration config) {
	24	VideoFrame frame = new VideoFrame(data);
	25	frame.config = config;
	26	frame.convert();
	27	return frame;
	28	}
	29
	30	private void convert() {
	31	/* TODO: how to do this?
	32	1) Resize to an image with the size of the number of leds and use config to define how many pixels deep into the screen to use.
	33	2) Resize to 16x9 and use 2 pixels of depth (or maybe 3) and interpolate for each led.
	34	3) Resize to 2 images where each led uses 2 pixels:
	35	vertical - 16 x <#leds>
	36	horizontal - <#leds> x 9
	37	4) Resize to cols x rows first, then resize to a vertical and a horizontal like in (3).
	38	*/
	39	Mat src = new Mat(config.video.height, config.video.width, CvType.CV_8UC2); // 8-bit, unsigned, 2 channels
	40	src.put(0, 0, data);
	41
	42	// Mat converted = new Mat();
	43	// Mat resized = new Mat();
	44	//
	45	// timeIt("total", () -> {
	46	// timeIt("yuyv2rgb", () -> Imgproc.cvtColor(src, converted, Imgproc.COLOR_YUV2RGB_YUYV)); // 3.5 - 4.0 ms
	47	// timeIt("resizing", () -> Imgproc.resize(converted, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, Imgproc.INTER_AREA)); // INTER_AREA is the best for shrinking, but also the slowest (~1.5 ms)
	48	// });
	49
	50	Mat cropped = src.submat(
	51	config.video.crop.top,
	52	config.video.height - config.video.crop.bottom,
	53	config.video.crop.left,
	54	config.video.width - config.video.crop.right
	55	);
	56	converted = new Mat();
	57	Imgproc.cvtColor(cropped, converted, config.video.format);
	58	// timeIt("model 1", () -> model1(converted, Imgproc.INTER_AREA));
	59	// timeIt("model 2", () -> model2(converted, Imgproc.INTER_AREA));
	60	// timeIt("model 3", () -> model3(converted, Imgproc.INTER_AREA));
	61	timeIt("model 4", () -> model4(converted, Imgproc.INTER_AREA));
	62	// save(converted, "/home/kaka/test-converted.data");
	63	// save(resized, "/home/kaka/test-resized.data");
	64	src.release();
	65	cropped.release();
	66	}
	67
	68	private void model1(Mat src, int interpolation) {
	69	Mat resized = new Mat();
	70	Imgproc.resize(src, resized, new Size(config.leds.cols, config.leds.rows), 0, 0, interpolation);
	71	}
	72
	73	private void model2(Mat src, int interpolation) {
	74	Mat resized = new Mat();
	75	Imgproc.resize(src, resized, new Size(16, 9), 0, 0, interpolation);
	76	}
	77
	78	private void model3(Mat src, int interpolation) {
	79	// colImage = new Mat();
	80	// rowImage = new Mat();
	81	// Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
	82	// Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
	83	}
	84
	85	private void model4(Mat src, int interpolation) {
	86	int width = 3 * src.cols() / 16;
	87	int height = 3 * src.rows() / 9;
	88	Mat[] cropped = new Mat[] {
	89	/* LEFT */ src.submat(0, src.rows(), 0, width),
	90	/* RIGHT */ src.submat(0, src.rows(), src.cols() - width, src.cols()),
	91	/* TOP */ src.submat(0, height, 0, src.cols()),
	92	/* BOTTOM */ src.submat(src.rows() - height, src.rows(), 0, src.cols()),
	93	};
	94	images = new Mat[] {new Mat(), new Mat(), new Mat(), new Mat()};
	95	// Imgproc.resize(cropped[ListPosition.LEFT.ordinal()], images[ListPosition.LEFT.ordinal()], new Size(3, config.leds.rows), 0, 0, interpolation);
	96	Imgproc.resize(cropped[0], images[0], new Size(3, config.leds.rows), 0, 0, interpolation);
	97	Imgproc.resize(cropped[1], images[1], new Size(3, config.leds.rows), 0, 0, interpolation);
	98	Imgproc.resize(cropped[2], images[2], new Size(config.leds.cols, 3), 0, 0, interpolation);
	99	Imgproc.resize(cropped[3], images[3], new Size(config.leds.cols, 3), 0, 0, interpolation);
	100	// Imgproc.resize(src, colImage, new Size(config.leds.cols, 9), 0, 0, interpolation);
	101	// Imgproc.resize(src, rowImage, new Size(16, config.leds.rows), 0, 0, interpolation);
	102	}
	103
	104	private Color wrappedGetLedColor(ListPosition listPosition, int xy) {
	105	Color c = getLedColor(listPosition, xy);
	106	double[] hsv = c.toHSV();
	107	double saturation = config.video.saturation >= 0.5
	108	? hsv[1] + (config.video.saturation - 0.5) * 2 * (1 - hsv[1])
	109	: hsv[1] - (1 - config.video.saturation * 2) * hsv[1];
	110	return Color.hsv(hsv[0], saturation, hsv[2]);
	111	}
	112
	113	private Color getLedColor(ListPosition listPosition, int xy) {
	114	// TODO: maybe use highest value from pixels? 100 % from 1st, 66 % from 2nd, 33 % from 3rd. colors might be strange.
	115	switch (listPosition) {
	116	case LEFT:
	117	return interpolatedRowColor(images[0], xy, 0, 1, 2);
	118	// return interpolatedRowColor(xy, 0, 1, 2);
	119	case RIGHT:
	120	return interpolatedRowColor(images[1], xy, 2, 1, 0);
	121	// return interpolatedRowColor(xy, 15, 14, 13);
	122	case TOP:
	123	return interpolatedColColor(images[2], xy, 0, 1, 2);
	124	// return interpolatedColColor(xy, 0, 1, 2);
	125	case BOTTOM:
	126	return interpolatedColColor(images[3], xy, 2, 1, 0);
	127	// return interpolatedColColor(xy, 8, 7, 6);
	128	}
	129	return null;
	130	}
	131
	132	// private Color interpolatedRowColor(int y, int x1, int x2, int x3) {
	133	private Color interpolatedRowColor(Mat rowImage, int y, int x1, int x2, int x3) {
	134	return pixelToColor(rowImage, x3, y).interpolate(pixelToColor(rowImage, x2, y), 0.65).interpolate(pixelToColor(rowImage, x1, y), 0.65);
	135	}
	136
	137	// private Color interpolatedColColor(int x, int y1, int y2, int y3) {
	138	private Color interpolatedColColor(Mat colImage, int x, int y1, int y2, int y3) {
	139	return pixelToColor(colImage, x, y3).interpolate(pixelToColor(colImage, x, y2), 0.65).interpolate(pixelToColor(colImage, x, y1), 0.65);
	140	}
	141
	142	private Color pixelToColor(Mat image, int x, int y) {
	143	byte[] rgb = new byte[3];
	144	image.get(y, x, rgb);
	145	return Color.rgb(rgb[0] & 0xff, rgb[1] & 0xff, rgb[2] & 0xff);
	146	}
	147
	148	private void save(Mat mat, String filepath) {
	149	byte[] data = new byte[mat.cols() * mat.rows() * mat.channels()];
	150	mat.get(0, 0, data);
	151	saveFile(data, filepath);
	152	}
	153
	154	public byte[] getData() {
	155	byte[] buff = new byte[(int) (converted.total() * converted.channels())];
	156	converted.get(0, 0, buff);
	157	return buff;
	158	}
	159
	160	// public Mat getColImage() {
	161	// return colImage;
	162	// }
	163
	164	// public Mat getRowImage() {
	165	// return rowImage;
	166	// }
	167
	168	public Mat getConvertedImage() {
	169	return converted;
	170	}
	171
	172	/**
	173	* Creates a LED frame going counter-clockwise from the bottom-left corner, sans the corners.
	174	*/
	175	public LedFrame getLedFrame() {
	176	LedFrame frame = LedFrame.from(config);
	177	int led = 0;
	178
	179	if (config.video.list.bottom)
	180	for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.BOTTOM, i));
	181	else
	182	for (int i = 0; i < config.leds.cols; i++) frame.setLedColor(led++, Color.BLACK);
	183
	184	if (config.video.list.right)
	185	for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.RIGHT, i));
	186	else
	187	for (int i = config.leds.rows - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);
	188
	189	if (config.video.list.top)
	190	for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.TOP, i));
	191	else
	192	for (int i = config.leds.cols - 1; i >= 0; i--) frame.setLedColor(led++, Color.BLACK);
	193
	194	if (config.video.list.left)
	195	for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, wrappedGetLedColor(ListPosition.LEFT, i));
	196	else
	197	for (int i = 0; i < config.leds.rows; i++) frame.setLedColor(led++, Color.BLACK);
	198
	199	return frame;
	200	}
	201
	202	private enum ListPosition {
	203	LEFT,
	204	RIGHT,
	205	TOP,
	206	BOTTOM
	207	}
	208	}