Check-in [e7128384f3]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:improvements in experimental jsmpeg SDL video driver
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:e7128384f3ddeb4bb2b7fa9306c433904bf53485
User & Date: chw 2019-02-12 06:15:24
Context
2019-02-13
18:09
add libwebsockets 2.0.3 (as in Debian 9.7) for undroidwish check-in: 8559fc1005 user: chw tags: trunk
2019-02-12
06:16
merge with trunk check-in: 68a876b70a user: chw tags: wtf-8-experiment
06:15
improvements in experimental jsmpeg SDL video driver check-in: e7128384f3 user: chw tags: trunk
2019-02-11
19:08
fixes in experimental jsmpeg SDL video driver check-in: 4fcdf8ee88 user: chw tags: trunk
Changes

Changes to jni/SDL2/src/video/jsmpeg/SDL_jsmpeg.c.

    38     38   #include <libavutil/imgutils.h>
    39     39   #include <libavcodec/avcodec.h>
    40     40   #include <libswscale/swscale.h>
    41     41   
    42     42   #include <libwebsockets.h>
    43     43   
    44     44   #include "SDL_jsmpeg_files.h"
           45  +
           46  +#if SDL_THREADS_DISABLED
           47  +#undef  USE_ENCODER_THREAD
           48  +#else
           49  +#define USE_ENCODER_THREAD
           50  +#endif
    45     51   
    46     52   /* Frame types, big endian */
    47     53   #define FRAME_TYPE_VIDEO 0x000001FA
    48     54   #define FRAME_TYPE_AUDIO 0x000001FB
    49     55   
    50     56   /* Input types, little endian */
    51     57   #define INPUT_KEY            0x0001
................................................................................
    97    103       AVCodecContext *context;
    98    104       AVFrame *frame;
    99    105       void *frame_buffer;
   100    106       struct SwsContext *sws;
   101    107       struct lws_context *lws;
   102    108       Client *clients;
   103    109       int ticks;
          110  +#ifdef USE_ENCODER_THREAD
          111  +    SDL_mutex *mutex;
          112  +    SDL_cond *cond;
          113  +    SDL_Thread *enc_thr;
          114  +    int busy;
          115  +    Frame *enc_frame;
          116  +#endif
   104    117   } SDL_WindowData;
   105    118   
   106    119   #define JSMPEG_DRIVER_NAME "jsmpeg"
   107    120   
   108    121   /* Functions */
   109    122   static int JSMPEG_VideoInit(_THIS);
          123  +static void JSMPEG_VideoQuit(_THIS);
   110    124   static void JSMPEG_GetDisplayModes(_THIS, SDL_VideoDisplay *display);
   111    125   static int JSMPEG_SetDisplayMode(_THIS, SDL_VideoDisplay *display, SDL_DisplayMode *mode);
   112         -static void JSMPEG_VideoQuit(_THIS);
          126  +static Frame *JSMPEG_EncodeFrame(SDL_WindowData *data);
          127  +static void JSMPEG_TransmitFrame(Frame *frame, Client *client);
          128  +#ifdef USE_ENCODER_THREAD
          129  +static int JSMPEG_EncoderThread(void *arg);
          130  +#endif
   113    131   static void JSMPEG_PumpEvents(_THIS);
   114    132   static int JSMPEG_CreateWindow(_THIS, SDL_Window *window);
   115    133   static void JSMPEG_DestroyWindow(_THIS, SDL_Window *window);
   116    134   static int JSMPEG_CreateWindowFramebuffer(_THIS, SDL_Window *window, Uint32 *format, void **pixels, int *pitch);
   117    135   static int JSMPEG_UpdateWindowFramebuffer(_THIS, SDL_Window *window, const SDL_Rect *rects, int numrects);
   118    136   static void JSMPEG_CleanupWindowData(SDL_WindowData *data);
   119    137   static void JSMPEG_DestroyWindowFramebuffer(_THIS, SDL_Window *window);
   120         -
   121    138   static int JSMPEG_CallbackHTTP(struct lws *lws, enum lws_callback_reasons reason, void *user, void *in, size_t len);
   122    139   static int JSMPEG_CallbackWS(struct lws *lws, enum lws_callback_reasons reason, void *user, void *in, size_t len);
   123    140   
   124    141   static struct lws_protocols LWS_protos[] = {
   125    142       { "http", JSMPEG_CallbackHTTP, sizeof(int), 0 },
   126    143       { "ws", JSMPEG_CallbackWS, sizeof(Client), 1024 * 1024 },
   127    144       { NULL, NULL, 0, 0 }
................................................................................
   535    552       if (mode->w >= 0 && mode->w <= 2048 &&
   536    553           mode->h >= 0 && mode->h <= 2048 &&
   537    554           mode->format == SDL_PIXELFORMAT_BGR888) {
   538    555           return 0;
   539    556       }
   540    557       return SDL_Unsupported();
   541    558   }
          559  +
          560  +static Frame *
          561  +JSMPEG_EncodeFrame(SDL_WindowData *data)
          562  +{
          563  +    AVPacket packet;
          564  +    Frame *frame = NULL;
          565  +    unsigned char *p;
          566  +    int ok = 0;
          567  +
          568  +    memset(&packet, 0, sizeof(packet));
          569  +    av_init_packet(&packet);
          570  +    avcodec_encode_video2(data->context, &packet, data->frame, &ok);
          571  +    if (ok) {
          572  +        frame = (Frame *) SDL_malloc(sizeof(Frame) + LWS_PRE + 8 + packet.size);
          573  +        if (frame != NULL) {
          574  +            frame->ref_count = 1;
          575  +            frame->size = packet.size + 8;
          576  +            frame->type = LWS_WRITE_BINARY;
          577  +            frame->data = frame + 1;
          578  +            p = (unsigned char *) frame->data;
          579  +            p += LWS_PRE;
          580  +            puti32(p + 0, FRAME_TYPE_VIDEO);
          581  +            puti32(p + 4, frame->size);
          582  +            memcpy(p + 8, packet.data, packet.size);
          583  +        }
          584  +    }
          585  +    av_free_packet(&packet);
          586  +    return frame;
          587  +}
          588  +
          589  +static void
          590  +JSMPEG_TransmitFrame(Frame *frame, Client *client)
          591  +{
          592  +    while (frame != NULL && client != NULL) {
          593  +        FrameRef *fref = SDL_calloc(1, sizeof(FrameRef));
          594  +
          595  +        if (fref != NULL) {
          596  +            fref->next = NULL;
          597  +            fref->frame = frame;
          598  +            frame->ref_count++;
          599  +            if (client->last != NULL) {
          600  +                client->last->next = fref;
          601  +            } else {
          602  +                client->last = client->first = fref;
          603  +                lws_callback_on_writable(client->socket);
          604  +            }
          605  +        }
          606  +        client = client->next;
          607  +    }
          608  +    if (frame != NULL && --frame->ref_count <= 0) {
          609  +        SDL_free(frame);
          610  +    }
          611  +}
          612  +
          613  +#ifdef USE_ENCODER_THREAD
          614  +static int
          615  +JSMPEG_EncoderThread(void *arg)
          616  +{
          617  +    SDL_WindowData *data = (SDL_WindowData *) arg;
          618  +
          619  +    SDL_LockMutex(data->mutex);
          620  +    while (data->busy >= 0) {
          621  +        if (data->busy > 0) {
          622  +            Frame *frame;
          623  +
          624  +            /* Encoding with mutex unlocked */
          625  +            SDL_UnlockMutex(data->mutex);
          626  +            frame = JSMPEG_EncodeFrame(data);
          627  +            SDL_LockMutex(data->mutex);
          628  +            if (frame != NULL) {
          629  +                if (data->enc_frame != NULL) {
          630  +                    if (--data->enc_frame->ref_count <= 0) {
          631  +                        SDL_free(frame);
          632  +                    }
          633  +                }
          634  +                data->enc_frame = frame;
          635  +            }
          636  +            data->busy = 0;
          637  +        }
          638  +        SDL_CondWait(data->cond, data->mutex);
          639  +    }
          640  +    SDL_UnlockMutex(data->mutex);
          641  +    return 0;
          642  +}
          643  +#endif
   542    644   
   543    645   static void
   544    646   JSMPEG_PumpEvents(_THIS)
   545    647   {
   546    648       SDL_WindowData *data = NULL;
   547    649       SDL_VideoData *c;
   548    650       int ticks;
   549    651   
   550    652       c = _this->driverdata;
   551    653       if (c != NULL) {
   552    654           data = c->data;
   553    655       }
   554    656       if (data != NULL) {
          657  +        Frame *frame = NULL;
          658  +        Client *client = data->clients;
   555    659   
   556         -        /* Handle events from websockets */
          660  +        /* Handle events from libwebsockets */
   557    661           lws_service(data->lws, 0);
          662  +
          663  +#ifdef USE_ENCODER_THREAD
          664  +        SDL_LockMutex(data->mutex);
          665  +        if (data->enc_frame != NULL) {
          666  +            frame = data->enc_frame;
          667  +            data->enc_frame = NULL;
          668  +        }
          669  +        SDL_UnlockMutex(data->mutex);
          670  +
          671  +        /* Transmission */
          672  +        JSMPEG_TransmitFrame(frame, client);
          673  +#endif
   558    674   
   559    675           ticks = SDL_GetTicks();
   560    676           /* About 15 frames per second */
   561    677           if (ticks - data->ticks >= 62) {
   562         -            Client *client = data->clients;
   563         -            Frame *frame = NULL;
   564         -            AVPacket packet;
   565    678               uint8_t *pixels[1];
   566    679               int linesizes[1];
   567         -            int ok = 0;
   568    680   
   569    681               data->ticks = ticks;
          682  +
   570    683               if (client == NULL) {
   571    684                   /* No client connected, do nothing */
   572    685                   return;
   573    686               }
   574         -
          687  +#ifdef USE_ENCODER_THREAD
          688  +            SDL_LockMutex(data->mutex);
          689  +            if (data->busy > 0) {
          690  +                /* Encoder thread still busy, do nothing */
          691  +                SDL_UnlockMutex(data->mutex);
          692  +                return;
          693  +            }
          694  +#endif
   575    695               /* Color space conversion */
   576    696               pixels[0] = (uint8_t *) data->surface->pixels;
   577    697               linesizes[0] = data->surface->pitch;
   578    698               sws_scale(data->sws, pixels, linesizes, 0, data->surface->h, data->frame->data, data->frame->linesize);
   579    699               data->frame->pts++;
   580    700   
          701  +#ifdef USE_ENCODER_THREAD
          702  +            /* Trigger encoding thread */
          703  +            data->busy = 1;
          704  +            SDL_CondSignal(data->cond);
          705  +            SDL_UnlockMutex(data->mutex);
          706  +#else
   581    707               /* Encoding */
   582         -            memset(&packet, 0, sizeof(packet));
   583         -            av_init_packet(&packet);
   584         -            avcodec_encode_video2(data->context, &packet, data->frame, &ok);
   585         -            if (ok) {
   586         -                unsigned char *p;
   587         -
   588         -                frame = (Frame *) SDL_malloc(sizeof(Frame) + LWS_PRE + 8 + packet.size);
   589         -                if (frame != NULL) {
   590         -                    frame->ref_count = 1;
   591         -                    frame->size = packet.size + 8;
   592         -                    frame->type = LWS_WRITE_BINARY;
   593         -                    frame->data = frame + 1;
   594         -                    p = (unsigned char *) frame->data;
   595         -                    p += LWS_PRE;
   596         -                    puti32(p + 0, FRAME_TYPE_VIDEO);
   597         -                    puti32(p + 4, frame->size);
   598         -                    memcpy(p + 8, packet.data, packet.size);
   599         -                }
   600         -            }
   601         -            av_free_packet(&packet);
          708  +            frame = JSMPEG_EncodeFrame(data);
   602    709   
   603    710               /* Transmission */
   604         -            while (frame != NULL && client != NULL) {
   605         -                FrameRef *fref = SDL_calloc(1, sizeof(FrameRef));
   606         -
   607         -                fref->next = NULL;
   608         -                fref->frame = frame;
   609         -                frame->ref_count++;
   610         -                if (client->last != NULL) {
   611         -                    client->last->next = fref;
   612         -                } else {
   613         -                    client->last = client->first = fref;
   614         -                    lws_callback_on_writable(client->socket);
   615         -                }
   616         -                client = client->next;
   617         -            }
   618         -            if (frame != NULL && --frame->ref_count <= 0) {
   619         -                SDL_free(frame);
   620         -            }
          711  +            JSMPEG_TransmitFrame(frame, client);
          712  +#endif
   621    713           }
   622    714       }
   623    715   }
   624    716   
   625    717   static int
   626    718   JSMPEG_CreateWindow(_THIS, SDL_Window *window)
   627    719   {
................................................................................
   744    836           info.port = SDL_atoi(env);
   745    837       }
   746    838       data->lws = lws_create_context(&info);
   747    839       if (data->lws == NULL) {
   748    840           JSMPEG_CleanupWindowData(data);
   749    841           return -1;
   750    842       }
          843  +
          844  +#ifdef USE_ENCODER_THREAD
          845  +    data->busy = 0;
          846  +    data->enc_frame = NULL;
          847  +    data->mutex = SDL_CreateMutex();
          848  +    data->cond = SDL_CreateCond();
          849  +    data->enc_thr = SDL_CreateThread(JSMPEG_EncoderThread, "jsmp-enc", data);
          850  +#endif
   751    851   
   752    852       /* Save the info and return. */
   753    853       *format = surface_format;
   754    854       *pixels = data->surface->pixels;
   755    855       *pitch = data->surface->pitch;
   756    856       return 0;
   757    857   }
................................................................................
   762    862       /* Nothing done here, real work is in JSMPEG_PumpEvents() */
   763    863       return 0;
   764    864   }
   765    865   
   766    866   static void
   767    867   JSMPEG_CleanupWindowData(SDL_WindowData *data)
   768    868   {
          869  +#ifdef USE_ENCODER_THREAD
          870  +    if (data->enc_thr != NULL) {
          871  +        SDL_LockMutex(data->mutex);
          872  +        data->busy = -1;
          873  +        SDL_CondSignal(data->cond);
          874  +        SDL_UnlockMutex(data->mutex);
          875  +        SDL_WaitThread(data->enc_thr, NULL);
          876  +        SDL_DestroyCond(data->cond);
          877  +        SDL_DestroyMutex(data->mutex);
          878  +        data->enc_thr = NULL;
          879  +        data->cond = NULL;
          880  +        data->mutex = NULL;
          881  +    }
          882  +    if (data->enc_frame != NULL) {
          883  +        SDL_free(data->enc_frame);
          884  +        data->enc_frame = NULL;
          885  +    }
          886  +#endif
   769    887       if (data->surface != NULL) {
   770    888           SDL_FreeSurface(data->surface);
   771    889           data->surface = NULL;
   772    890       }
   773    891       if (data->sws != NULL) {
   774    892           sws_freeContext(data->sws);
   775    893           data->sws = NULL;
................................................................................
   897   1015           int type, flags;
   898   1016   
   899   1017           type = geti16(p + 0);
   900   1018           if (type & INPUT_KEY) {
   901   1019               if (len >= 6) {
   902   1020                   flags = geti16(p + 2);
   903   1021                   if (flags & KEY_PRESS) {
   904         -                    int codepoint;
         1022  +                    int hi, lo;
   905   1023                       char text[5];
   906   1024   
   907         -                    codepoint = geti16(p + 4) | (geti16(p + 6) << 16);
   908         -                    if (convUTF32toUTF8(codepoint, text)) {
         1025  +                    hi = geti16(p + 4);
         1026  +                    if (hi >= 0xD800 && hi < 0xDC00) {
         1027  +                        if (len >= 8) {
         1028  +                            lo = geti16(p + 6);
         1029  +                        } else {
         1030  +                            lo = 0;
         1031  +                        }
         1032  +                        if (lo >= 0xDC00 && lo < 0xE000) {
         1033  +                            hi = (((hi & 0x03FF) << 10) | (lo & 0x03FF)) + 0x10000;
         1034  +                        } else {
         1035  +                            hi = 0;
         1036  +                        }
         1037  +                    }
         1038  +                    if (hi && convUTF32toUTF8(hi, text)) {
   909   1039                           SDL_SendKeyboardText(text);
   910   1040                       }
   911   1041                   } else {
   912   1042                       int scancode, key_code = geti16(p + 4);
   913   1043   
   914   1044                       if (key_code < SDL_arraysize(scancode_table)) {
   915   1045                           scancode = scancode_table[key_code];

Changes to jni/SDL2/src/video/jsmpeg/data/jsmpg-vnc.js.

    44     44   	     key == 37 ||		// left
    45     45   	     key == 38 ||		// up
    46     46   	     key == 39 ||		// right
    47     47   	     key == 40 ) {		// down
    48     48   		// don't prevent
    49     49   	} else {
    50     50   		if ( action == KEY_DOWN && ev.key.length > 0 && ev.key.length <= 2 ) {
    51         -			var lo = ev.key.charCodeAt(0);
    52         -			var hi = 0;
    53         -			if ( lo >= 0xD800 && lo < 0xDC00 ) {
    54         -				if (ev.key.length > 1) {
    55         -					hi = ev.key.charCodeAt(1);
           51  +			var hi = ev.key.charCodeAt(0);
           52  +			var lo = 0;
           53  +			if ( hi >= 0xD800 && hi < 0xDC00 ) {
           54  +				if ( ev.key.length > 1 ) {
           55  +					lo = ev.key.charCodeAt(1);
    56     56   				}
    57     57   			}
    58         -			client.send(new Uint16Array([INPUT_KEY, KEY_PRESS, lo, hi]));
           58  +			client.send(new Uint16Array([INPUT_KEY, KEY_PRESS, hi, lo]));
    59     59   		}
    60     60   		ev.preventDefault();
    61     61   	}
    62     62   	if ( action == KEY_UP ) {
    63     63   		ev.stopPropagation();
    64     64   	}
    65     65   };