Check-in [e7128384f3]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:improvements in experimental jsmpeg SDL video driver
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:e7128384f3ddeb4bb2b7fa9306c433904bf53485
User & Date: chw 2019-02-12 06:15:24
Context
2019-02-13
18:09
add libwebsockets 2.0.3 (as in Debian 9.7) for undroidwish check-in: 8559fc1005 user: chw tags: trunk
2019-02-12
06:16
merge with trunk check-in: 68a876b70a user: chw tags: wtf-8-experiment
06:15
improvements in experimental jsmpeg SDL video driver check-in: e7128384f3 user: chw tags: trunk
2019-02-11
19:08
fixes in experimental jsmpeg SDL video driver check-in: 4fcdf8ee88 user: chw tags: trunk
Changes

Changes to jni/SDL2/src/video/jsmpeg/SDL_jsmpeg.c.

38
39
40
41
42
43
44






45
46
47
48
49
50
51
..
97
98
99
100
101
102
103







104
105
106
107
108
109

110
111

112



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
...
535
536
537
538
539
540
541





















































































542
543
544
545
546
547
548
549
550
551
552
553
554


555
556
557












558
559
560
561
562
563
564
565
566
567
568
569

570
571
572
573






574

575
576
577
578
579
580






581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620


621
622
623
624
625
626
627
...
744
745
746
747
748
749
750








751
752
753
754
755
756
757
...
762
763
764
765
766
767
768


















769
770
771
772
773
774
775
...
897
898
899
900
901
902
903
904
905
906
907












908
909
910
911
912
913
914
915
#include <libavutil/imgutils.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>

#include <libwebsockets.h>

#include "SDL_jsmpeg_files.h"







/* Frame types, big endian */
#define FRAME_TYPE_VIDEO 0x000001FA
#define FRAME_TYPE_AUDIO 0x000001FB

/* Input types, little endian */
#define INPUT_KEY            0x0001
................................................................................
    AVCodecContext *context;
    AVFrame *frame;
    void *frame_buffer;
    struct SwsContext *sws;
    struct lws_context *lws;
    Client *clients;
    int ticks;







} SDL_WindowData;

#define JSMPEG_DRIVER_NAME "jsmpeg"

/* Functions */
static int JSMPEG_VideoInit(_THIS);

static void JSMPEG_GetDisplayModes(_THIS, SDL_VideoDisplay *display);
static int JSMPEG_SetDisplayMode(_THIS, SDL_VideoDisplay *display, SDL_DisplayMode *mode);

static void JSMPEG_VideoQuit(_THIS);



static void JSMPEG_PumpEvents(_THIS);
static int JSMPEG_CreateWindow(_THIS, SDL_Window *window);
static void JSMPEG_DestroyWindow(_THIS, SDL_Window *window);
static int JSMPEG_CreateWindowFramebuffer(_THIS, SDL_Window *window, Uint32 *format, void **pixels, int *pitch);
static int JSMPEG_UpdateWindowFramebuffer(_THIS, SDL_Window *window, const SDL_Rect *rects, int numrects);
static void JSMPEG_CleanupWindowData(SDL_WindowData *data);
static void JSMPEG_DestroyWindowFramebuffer(_THIS, SDL_Window *window);

static int JSMPEG_CallbackHTTP(struct lws *lws, enum lws_callback_reasons reason, void *user, void *in, size_t len);
static int JSMPEG_CallbackWS(struct lws *lws, enum lws_callback_reasons reason, void *user, void *in, size_t len);

static struct lws_protocols LWS_protos[] = {
    { "http", JSMPEG_CallbackHTTP, sizeof(int), 0 },
    { "ws", JSMPEG_CallbackWS, sizeof(Client), 1024 * 1024 },
    { NULL, NULL, 0, 0 }
................................................................................
    if (mode->w >= 0 && mode->w <= 2048 &&
        mode->h >= 0 && mode->h <= 2048 &&
        mode->format == SDL_PIXELFORMAT_BGR888) {
        return 0;
    }
    return SDL_Unsupported();
}






















































































static void
JSMPEG_PumpEvents(_THIS)
{
    SDL_WindowData *data = NULL;
    SDL_VideoData *c;
    int ticks;

    c = _this->driverdata;
    if (c != NULL) {
        data = c->data;
    }
    if (data != NULL) {



        /* Handle events from websockets */
        lws_service(data->lws, 0);













        ticks = SDL_GetTicks();
        /* About 15 frames per second */
        if (ticks - data->ticks >= 62) {
            Client *client = data->clients;
            Frame *frame = NULL;
            AVPacket packet;
            uint8_t *pixels[1];
            int linesizes[1];
            int ok = 0;

            data->ticks = ticks;

            if (client == NULL) {
                /* No client connected, do nothing */
                return;
            }








            /* Color space conversion */
            pixels[0] = (uint8_t *) data->surface->pixels;
            linesizes[0] = data->surface->pitch;
            sws_scale(data->sws, pixels, linesizes, 0, data->surface->h, data->frame->data, data->frame->linesize);
            data->frame->pts++;







            /* Encoding */
            memset(&packet, 0, sizeof(packet));
            av_init_packet(&packet);
            avcodec_encode_video2(data->context, &packet, data->frame, &ok);
            if (ok) {
                unsigned char *p;

                frame = (Frame *) SDL_malloc(sizeof(Frame) + LWS_PRE + 8 + packet.size);
                if (frame != NULL) {
                    frame->ref_count = 1;
                    frame->size = packet.size + 8;
                    frame->type = LWS_WRITE_BINARY;
                    frame->data = frame + 1;
                    p = (unsigned char *) frame->data;
                    p += LWS_PRE;
                    puti32(p + 0, FRAME_TYPE_VIDEO);
                    puti32(p + 4, frame->size);
                    memcpy(p + 8, packet.data, packet.size);
                }
            }
            av_free_packet(&packet);

            /* Transmission */
            while (frame != NULL && client != NULL) {
                FrameRef *fref = SDL_calloc(1, sizeof(FrameRef));

                fref->next = NULL;
                fref->frame = frame;
                frame->ref_count++;
                if (client->last != NULL) {
                    client->last->next = fref;
                } else {
                    client->last = client->first = fref;
                    lws_callback_on_writable(client->socket);
                }
                client = client->next;
            }
            if (frame != NULL && --frame->ref_count <= 0) {
                SDL_free(frame);
            }


        }
    }
}

static int
JSMPEG_CreateWindow(_THIS, SDL_Window *window)
{
................................................................................
        info.port = SDL_atoi(env);
    }
    data->lws = lws_create_context(&info);
    if (data->lws == NULL) {
        JSMPEG_CleanupWindowData(data);
        return -1;
    }









    /* Save the info and return. */
    *format = surface_format;
    *pixels = data->surface->pixels;
    *pitch = data->surface->pitch;
    return 0;
}
................................................................................
    /* Nothing done here, real work is in JSMPEG_PumpEvents() */
    return 0;
}

static void
JSMPEG_CleanupWindowData(SDL_WindowData *data)
{


















    if (data->surface != NULL) {
        SDL_FreeSurface(data->surface);
        data->surface = NULL;
    }
    if (data->sws != NULL) {
        sws_freeContext(data->sws);
        data->sws = NULL;
................................................................................
        int type, flags;

        type = geti16(p + 0);
        if (type & INPUT_KEY) {
            if (len >= 6) {
                flags = geti16(p + 2);
                if (flags & KEY_PRESS) {
                    int codepoint;
                    char text[5];

                    codepoint = geti16(p + 4) | (geti16(p + 6) << 16);












                    if (convUTF32toUTF8(codepoint, text)) {
                        SDL_SendKeyboardText(text);
                    }
                } else {
                    int scancode, key_code = geti16(p + 4);

                    if (key_code < SDL_arraysize(scancode_table)) {
                        scancode = scancode_table[key_code];







>
>
>
>
>
>







 







>
>
>
>
>
>
>






>


>
|
>
>
>







<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













>
>

|

>
>
>
>
>
>
>
>
>
>
>
>




<
<
<


<


>




>
>
>
>
>
>
|
>






>
>
>
>
>
>

<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
|
<
<
<

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
>
>







 







>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|


|
>
>
>
>
>
>
>
>
>
>
>
>
|







38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
...
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

138
139
140
141
142
143
144
...
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677



678
679

680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707






708










709



710

















711
712
713
714
715
716
717
718
719
...
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
...
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
....
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
#include <libavutil/imgutils.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>

#include <libwebsockets.h>

#include "SDL_jsmpeg_files.h"

#if SDL_THREADS_DISABLED
#undef  USE_ENCODER_THREAD
#else
#define USE_ENCODER_THREAD
#endif

/* Frame types, big endian */
#define FRAME_TYPE_VIDEO 0x000001FA
#define FRAME_TYPE_AUDIO 0x000001FB

/* Input types, little endian */
#define INPUT_KEY            0x0001
................................................................................
    AVCodecContext *context;
    AVFrame *frame;
    void *frame_buffer;
    struct SwsContext *sws;
    struct lws_context *lws;
    Client *clients;
    int ticks;
#ifdef USE_ENCODER_THREAD
    SDL_mutex *mutex;
    SDL_cond *cond;
    SDL_Thread *enc_thr;
    int busy;
    Frame *enc_frame;
#endif
} SDL_WindowData;

#define JSMPEG_DRIVER_NAME "jsmpeg"

/* Functions */
static int JSMPEG_VideoInit(_THIS);
static void JSMPEG_VideoQuit(_THIS);
static void JSMPEG_GetDisplayModes(_THIS, SDL_VideoDisplay *display);
static int JSMPEG_SetDisplayMode(_THIS, SDL_VideoDisplay *display, SDL_DisplayMode *mode);
static Frame *JSMPEG_EncodeFrame(SDL_WindowData *data);
static void JSMPEG_TransmitFrame(Frame *frame, Client *client);
#ifdef USE_ENCODER_THREAD
static int JSMPEG_EncoderThread(void *arg);
#endif
static void JSMPEG_PumpEvents(_THIS);
static int JSMPEG_CreateWindow(_THIS, SDL_Window *window);
static void JSMPEG_DestroyWindow(_THIS, SDL_Window *window);
static int JSMPEG_CreateWindowFramebuffer(_THIS, SDL_Window *window, Uint32 *format, void **pixels, int *pitch);
static int JSMPEG_UpdateWindowFramebuffer(_THIS, SDL_Window *window, const SDL_Rect *rects, int numrects);
static void JSMPEG_CleanupWindowData(SDL_WindowData *data);
static void JSMPEG_DestroyWindowFramebuffer(_THIS, SDL_Window *window);

static int JSMPEG_CallbackHTTP(struct lws *lws, enum lws_callback_reasons reason, void *user, void *in, size_t len);
static int JSMPEG_CallbackWS(struct lws *lws, enum lws_callback_reasons reason, void *user, void *in, size_t len);

static struct lws_protocols LWS_protos[] = {
    { "http", JSMPEG_CallbackHTTP, sizeof(int), 0 },
    { "ws", JSMPEG_CallbackWS, sizeof(Client), 1024 * 1024 },
    { NULL, NULL, 0, 0 }
................................................................................
    if (mode->w >= 0 && mode->w <= 2048 &&
        mode->h >= 0 && mode->h <= 2048 &&
        mode->format == SDL_PIXELFORMAT_BGR888) {
        return 0;
    }
    return SDL_Unsupported();
}

static Frame *
JSMPEG_EncodeFrame(SDL_WindowData *data)
{
    AVPacket packet;
    Frame *frame = NULL;
    unsigned char *p;
    int ok = 0;

    memset(&packet, 0, sizeof(packet));
    av_init_packet(&packet);
    avcodec_encode_video2(data->context, &packet, data->frame, &ok);
    if (ok) {
        frame = (Frame *) SDL_malloc(sizeof(Frame) + LWS_PRE + 8 + packet.size);
        if (frame != NULL) {
            frame->ref_count = 1;
            frame->size = packet.size + 8;
            frame->type = LWS_WRITE_BINARY;
            frame->data = frame + 1;
            p = (unsigned char *) frame->data;
            p += LWS_PRE;
            puti32(p + 0, FRAME_TYPE_VIDEO);
            puti32(p + 4, frame->size);
            memcpy(p + 8, packet.data, packet.size);
        }
    }
    av_free_packet(&packet);
    return frame;
}

static void
JSMPEG_TransmitFrame(Frame *frame, Client *client)
{
    while (frame != NULL && client != NULL) {
        FrameRef *fref = SDL_calloc(1, sizeof(FrameRef));

        if (fref != NULL) {
            fref->next = NULL;
            fref->frame = frame;
            frame->ref_count++;
            if (client->last != NULL) {
                client->last->next = fref;
            } else {
                client->last = client->first = fref;
                lws_callback_on_writable(client->socket);
            }
        }
        client = client->next;
    }
    if (frame != NULL && --frame->ref_count <= 0) {
        SDL_free(frame);
    }
}

#ifdef USE_ENCODER_THREAD
static int
JSMPEG_EncoderThread(void *arg)
{
    SDL_WindowData *data = (SDL_WindowData *) arg;

    SDL_LockMutex(data->mutex);
    while (data->busy >= 0) {
        if (data->busy > 0) {
            Frame *frame;

            /* Encoding with mutex unlocked */
            SDL_UnlockMutex(data->mutex);
            frame = JSMPEG_EncodeFrame(data);
            SDL_LockMutex(data->mutex);
            if (frame != NULL) {
                if (data->enc_frame != NULL) {
                    if (--data->enc_frame->ref_count <= 0) {
                        SDL_free(frame);
                    }
                }
                data->enc_frame = frame;
            }
            data->busy = 0;
        }
        SDL_CondWait(data->cond, data->mutex);
    }
    SDL_UnlockMutex(data->mutex);
    return 0;
}
#endif

static void
JSMPEG_PumpEvents(_THIS)
{
    SDL_WindowData *data = NULL;
    SDL_VideoData *c;
    int ticks;

    c = _this->driverdata;
    if (c != NULL) {
        data = c->data;
    }
    if (data != NULL) {
        Frame *frame = NULL;
        Client *client = data->clients;

        /* Handle events from libwebsockets */
        lws_service(data->lws, 0);

#ifdef USE_ENCODER_THREAD
        SDL_LockMutex(data->mutex);
        if (data->enc_frame != NULL) {
            frame = data->enc_frame;
            data->enc_frame = NULL;
        }
        SDL_UnlockMutex(data->mutex);

        /* Transmission */
        JSMPEG_TransmitFrame(frame, client);
#endif

        ticks = SDL_GetTicks();
        /* About 15 frames per second */
        if (ticks - data->ticks >= 62) {



            uint8_t *pixels[1];
            int linesizes[1];


            data->ticks = ticks;

            if (client == NULL) {
                /* No client connected, do nothing */
                return;
            }
#ifdef USE_ENCODER_THREAD
            SDL_LockMutex(data->mutex);
            if (data->busy > 0) {
                /* Encoder thread still busy, do nothing */
                SDL_UnlockMutex(data->mutex);
                return;
            }
#endif
            /* Color space conversion */
            pixels[0] = (uint8_t *) data->surface->pixels;
            linesizes[0] = data->surface->pitch;
            sws_scale(data->sws, pixels, linesizes, 0, data->surface->h, data->frame->data, data->frame->linesize);
            data->frame->pts++;

#ifdef USE_ENCODER_THREAD
            /* Trigger encoding thread */
            data->busy = 1;
            SDL_CondSignal(data->cond);
            SDL_UnlockMutex(data->mutex);
#else
            /* Encoding */






            frame = JSMPEG_EncodeFrame(data);














            /* Transmission */

















            JSMPEG_TransmitFrame(frame, client);
#endif
        }
    }
}

static int
JSMPEG_CreateWindow(_THIS, SDL_Window *window)
{
................................................................................
        info.port = SDL_atoi(env);
    }
    data->lws = lws_create_context(&info);
    if (data->lws == NULL) {
        JSMPEG_CleanupWindowData(data);
        return -1;
    }

#ifdef USE_ENCODER_THREAD
    data->busy = 0;
    data->enc_frame = NULL;
    data->mutex = SDL_CreateMutex();
    data->cond = SDL_CreateCond();
    data->enc_thr = SDL_CreateThread(JSMPEG_EncoderThread, "jsmp-enc", data);
#endif

    /* Save the info and return. */
    *format = surface_format;
    *pixels = data->surface->pixels;
    *pitch = data->surface->pitch;
    return 0;
}
................................................................................
    /* Nothing done here, real work is in JSMPEG_PumpEvents() */
    return 0;
}

static void
JSMPEG_CleanupWindowData(SDL_WindowData *data)
{
#ifdef USE_ENCODER_THREAD
    if (data->enc_thr != NULL) {
        SDL_LockMutex(data->mutex);
        data->busy = -1;
        SDL_CondSignal(data->cond);
        SDL_UnlockMutex(data->mutex);
        SDL_WaitThread(data->enc_thr, NULL);
        SDL_DestroyCond(data->cond);
        SDL_DestroyMutex(data->mutex);
        data->enc_thr = NULL;
        data->cond = NULL;
        data->mutex = NULL;
    }
    if (data->enc_frame != NULL) {
        SDL_free(data->enc_frame);
        data->enc_frame = NULL;
    }
#endif
    if (data->surface != NULL) {
        SDL_FreeSurface(data->surface);
        data->surface = NULL;
    }
    if (data->sws != NULL) {
        sws_freeContext(data->sws);
        data->sws = NULL;
................................................................................
        int type, flags;

        type = geti16(p + 0);
        if (type & INPUT_KEY) {
            if (len >= 6) {
                flags = geti16(p + 2);
                if (flags & KEY_PRESS) {
                    int hi, lo;
                    char text[5];

                    hi = geti16(p + 4);
                    if (hi >= 0xD800 && hi < 0xDC00) {
                        if (len >= 8) {
                            lo = geti16(p + 6);
                        } else {
                            lo = 0;
                        }
                        if (lo >= 0xDC00 && lo < 0xE000) {
                            hi = (((hi & 0x03FF) << 10) | (lo & 0x03FF)) + 0x10000;
                        } else {
                            hi = 0;
                        }
                    }
                    if (hi && convUTF32toUTF8(hi, text)) {
                        SDL_SendKeyboardText(text);
                    }
                } else {
                    int scancode, key_code = geti16(p + 4);

                    if (key_code < SDL_arraysize(scancode_table)) {
                        scancode = scancode_table[key_code];

Changes to jni/SDL2/src/video/jsmpeg/data/jsmpg-vnc.js.

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
	     key == 37 ||		// left
	     key == 38 ||		// up
	     key == 39 ||		// right
	     key == 40 ) {		// down
		// don't prevent
	} else {
		if ( action == KEY_DOWN && ev.key.length > 0 && ev.key.length <= 2 ) {
			var lo = ev.key.charCodeAt(0);
			var hi = 0;
			if ( lo >= 0xD800 && lo < 0xDC00 ) {
				if (ev.key.length > 1) {
					hi = ev.key.charCodeAt(1);
				}
			}
			client.send(new Uint16Array([INPUT_KEY, KEY_PRESS, lo, hi]));
		}
		ev.preventDefault();
	}
	if ( action == KEY_UP ) {
		ev.stopPropagation();
	}
};







|
|
|
|
|


|







44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
	     key == 37 ||		// left
	     key == 38 ||		// up
	     key == 39 ||		// right
	     key == 40 ) {		// down
		// don't prevent
	} else {
		if ( action == KEY_DOWN && ev.key.length > 0 && ev.key.length <= 2 ) {
			var hi = ev.key.charCodeAt(0);
			var lo = 0;
			if ( hi >= 0xD800 && hi < 0xDC00 ) {
				if ( ev.key.length > 1 ) {
					lo = ev.key.charCodeAt(1);
				}
			}
			client.send(new Uint16Array([INPUT_KEY, KEY_PRESS, hi, lo]));
		}
		ev.preventDefault();
	}
	if ( action == KEY_UP ) {
		ev.stopPropagation();
	}
};