MediaPlayer: support external timed text in java

Bug: 16385674
Change-Id: I7c2bf7a7d88c8396c3e228e3cf500998a3fa9db8

6 files changed, 343 insertions, 21 deletions

diff --git a/media/java/android/media/MediaFormat.java b/media/java/android/media/MediaFormat.java
index a1ccf60..c6586c0 100644
--- a/media/java/android/media/MediaFormat.java
+++ b/media/java/android/media/MediaFormat.java
@@ -483,6 +483,9 @@ public final class MediaFormat {
      */
     public static final String KEY_IS_FORCED_SUBTITLE = "is-forced-subtitle";
 
+    /** @hide */
+    public static final String KEY_IS_TIMED_TEXT = "is-timed-text";
+
     /* package private */ MediaFormat(Map<String, Object> map) {
         mMap = map;
     }
diff --git a/media/java/android/media/MediaPlayer.java b/media/java/android/media/MediaPlayer.java
index d9217a0..6605a98 100644
--- a/media/java/android/media/MediaPlayer.java
+++ b/media/java/android/media/MediaPlayer.java
@@ -18,6 +18,7 @@ package android.media;
 
 import android.app.ActivityThread;
 import android.app.AppOpsManager;
+import android.app.Application;
 import android.content.BroadcastReceiver;
 import android.content.ContentResolver;
 import android.content.Context;
@@ -36,6 +37,8 @@ import android.os.Process;
 import android.os.PowerManager;
 import android.os.RemoteException;
 import android.os.ServiceManager;
+import android.system.ErrnoException;
+import android.system.OsConstants;
 import android.util.Log;
 import android.view.Surface;
 import android.view.SurfaceHolder;
@@ -44,15 +47,22 @@ import android.media.AudioManager;
 import android.media.MediaFormat;
 import android.media.MediaTimeProvider;
 import android.media.SubtitleController;
+import android.media.SubtitleController.Anchor;
 import android.media.SubtitleData;
+import android.media.SubtitleTrack.RenderingWidget;
 
 import com.android.internal.app.IAppOpsService;
 
+import libcore.io.IoBridge;
+import libcore.io.Libcore;
+
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.lang.Runnable;
 import java.net.InetSocketAddress;
 import java.util.Map;
@@ -1846,7 +1856,10 @@ public class MediaPlayer implements SubtitleController.Listener
         System.arraycopy(trackInfo, 0, allTrackInfo, 0, trackInfo.length);
         int i = trackInfo.length;
         for (SubtitleTrack track: mOutOfBandSubtitleTracks) {
-            allTrackInfo[i] = new TrackInfo(TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE, track.getFormat());
+            int type = track.isTimedText()
+                    ? TrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT
+                    : TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE;
+            allTrackInfo[i] = new TrackInfo(type, track.getFormat());
             ++i;
         }
         return allTrackInfo;
@@ -1891,7 +1904,7 @@ public class MediaPlayer implements SubtitleController.Listener
      * A helper function to check if the mime type is supported by media framework.
      */
     private static boolean availableMimeTypeForExternalSource(String mimeType) {
-        if (mimeType == MEDIA_MIMETYPE_TEXT_SUBRIP) {
+        if (MEDIA_MIMETYPE_TEXT_SUBRIP.equals(mimeType)) {
             return true;
         }
         return false;
@@ -2147,27 +2160,97 @@ public class MediaPlayer implements SubtitleController.Listener
      * @throws IllegalArgumentException if the mimeType is not supported.
      * @throws IllegalStateException if called in an invalid state.
      */
-    public void addTimedTextSource(FileDescriptor fd, long offset, long length, String mimeType)
+    public void addTimedTextSource(FileDescriptor fd, long offset, long length, String mime)
             throws IllegalArgumentException, IllegalStateException {
-        if (!availableMimeTypeForExternalSource(mimeType)) {
-            throw new IllegalArgumentException("Illegal mimeType for timed text source: " + mimeType);
-
+        if (!availableMimeTypeForExternalSource(mime)) {
+            throw new IllegalArgumentException("Illegal mimeType for timed text source: " + mime);
         }
 
-        Parcel request = Parcel.obtain();
-        Parcel reply = Parcel.obtain();
+        FileDescriptor fd2;
         try {
-            request.writeInterfaceToken(IMEDIA_PLAYER);
-            request.writeInt(INVOKE_ID_ADD_EXTERNAL_SOURCE_FD);
-            request.writeFileDescriptor(fd);
-            request.writeLong(offset);
-            request.writeLong(length);
-            request.writeString(mimeType);
-            invoke(request, reply);
-        } finally {
-            request.recycle();
-            reply.recycle();
+            fd2 = Libcore.os.dup(fd);
+        } catch (ErrnoException ex) {
+            Log.e(TAG, ex.getMessage(), ex);
+            throw new RuntimeException(ex);
+        }
+
+        final MediaFormat fFormat = new MediaFormat();
+        fFormat.setString(MediaFormat.KEY_MIME, mime);
+        fFormat.setInteger(MediaFormat.KEY_IS_TIMED_TEXT, 1);
+
+        Context context = ActivityThread.currentApplication();
+        // A MediaPlayer created by a VideoView should already have its mSubtitleController set.
+        if (mSubtitleController == null) {
+            mSubtitleController = new SubtitleController(context, mTimeProvider, this);
+            mSubtitleController.setAnchor(new Anchor() {
+                @Override
+                public void setSubtitleWidget(RenderingWidget subtitleWidget) {
+                }
+
+                @Override
+                public Looper getSubtitleLooper() {
+                    return Looper.getMainLooper();
+                }
+            });
         }
+
+        if (!mSubtitleController.hasRendererFor(fFormat)) {
+            // test and add not atomic
+            mSubtitleController.registerRenderer(new SRTRenderer(context, mEventHandler));
+        }
+        final SubtitleTrack track = mSubtitleController.addTrack(fFormat);
+        mOutOfBandSubtitleTracks.add(track);
+
+        final FileDescriptor fd3 = fd2;
+        final long offset2 = offset;
+        final long length2 = length;
+        final HandlerThread thread = new HandlerThread(
+                "TimedTextReadThread",
+                Process.THREAD_PRIORITY_BACKGROUND + Process.THREAD_PRIORITY_MORE_FAVORABLE);
+        thread.start();
+        Handler handler = new Handler(thread.getLooper());
+        handler.post(new Runnable() {
+            private int addTrack() {
+                InputStream is = null;
+                final ByteArrayOutputStream bos = new ByteArrayOutputStream();
+                try {
+                    Libcore.os.lseek(fd3, offset2, OsConstants.SEEK_SET);
+                    byte[] buffer = new byte[4096];
+                    for (int total = 0; total < length2;) {
+                        int remain = (int)length2 - total;
+                        int bytes = IoBridge.read(fd3, buffer, 0, Math.min(buffer.length, remain));
+                        if (bytes < 0) {
+                            break;
+                        } else {
+                            bos.write(buffer, 0, bytes);
+                            total += bytes;
+                        }
+                    }
+                    track.onData(bos.toByteArray(), true /* eos */, ~0 /* runID: keep forever */);
+                    return MEDIA_INFO_EXTERNAL_METADATA_UPDATE;
+                } catch (Exception e) {
+                    Log.e(TAG, e.getMessage(), e);
+                    return MEDIA_INFO_TIMED_TEXT_ERROR;
+                } finally {
+                    if (is != null) {
+                        try {
+                            is.close();
+                        } catch (IOException e) {
+                            Log.e(TAG, e.getMessage(), e);
+                        }
+                    }
+                }
+            }
+
+            public void run() {
+                int res = addTrack();
+                if (mEventHandler != null) {
+                    Message m = mEventHandler.obtainMessage(MEDIA_INFO, res, 0, null);
+                    mEventHandler.sendMessage(m);
+                }
+                thread.getLooper().quitSafely();
+            }
+        });
     }
 
     /**
@@ -2275,6 +2358,13 @@ public class MediaPlayer implements SubtitleController.Listener
 
         if (mSubtitleController != null && track != null) {
             if (select) {
+                if (track.isTimedText()) {
+                    int ttIndex = getSelectedTrack(TrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT);
+                    if (ttIndex >= 0 && ttIndex < mInbandSubtitleTracks.length) {
+                        // deselect inband counterpart
+                        selectOrDeselectInbandTrack(ttIndex, false);
+                    }
+                }
                 mSubtitleController.selectTrack(track);
             } else if (mSubtitleController.getSelectedTrack() == track) {
                 mSubtitleController.selectTrack(null);
diff --git a/media/java/android/media/SRTRenderer.java b/media/java/android/media/SRTRenderer.java
new file mode 100644
index 0000000..ee4edee
--- /dev/null
+++ b/media/java/android/media/SRTRenderer.java
@@ -0,0 +1,202 @@
+package android.media;
+
+import android.content.Context;
+import android.media.SubtitleController.Renderer;
+import android.os.Handler;
+import android.os.Message;
+import android.os.Parcel;
+import android.util.Log;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Vector;
+
+/** @hide */
+public class SRTRenderer extends Renderer {
+    private final Context mContext;
+    private final boolean mRender;
+    private final Handler mEventHandler;
+
+    private WebVttRenderingWidget mRenderingWidget;
+
+    public SRTRenderer(Context context) {
+        this(context, null);
+    }
+
+    SRTRenderer(Context mContext, Handler mEventHandler) {
+        this.mContext = mContext;
+        this.mRender = (mEventHandler == null);
+        this.mEventHandler = mEventHandler;
+    }
+
+    @Override
+    public boolean supports(MediaFormat format) {
+        if (format.containsKey(MediaFormat.KEY_MIME)) {
+            if (!format.getString(MediaFormat.KEY_MIME)
+                    .equals(MediaPlayer.MEDIA_MIMETYPE_TEXT_SUBRIP)) {
+                return false;
+            };
+            return mRender == (format.getInteger(MediaFormat.KEY_IS_TIMED_TEXT, 0) == 0);
+        }
+        return false;
+    }
+
+    @Override
+    public SubtitleTrack createTrack(MediaFormat format) {
+        if (mRender && mRenderingWidget == null) {
+            mRenderingWidget = new WebVttRenderingWidget(mContext);
+        }
+
+        if (mRender) {
+            return new SRTTrack(mRenderingWidget, format);
+        } else {
+            return new SRTTrack(mEventHandler, format);
+        }
+    }
+}
+
+class SRTTrack extends WebVttTrack {
+    private static final int MEDIA_TIMED_TEXT = 99;   // MediaPlayer.MEDIA_TIMED_TEXT
+    private static final int KEY_STRUCT_TEXT = 16;    // TimedText.KEY_STRUCT_TEXT
+    private static final int KEY_START_TIME = 7;      // TimedText.KEY_START_TIME
+    private static final int KEY_LOCAL_SETTING = 102; // TimedText.KEY_START_TIME
+
+    private static final String TAG = "SRTTrack";
+    private final Handler mEventHandler;
+
+    SRTTrack(WebVttRenderingWidget renderingWidget, MediaFormat format) {
+        super(renderingWidget, format);
+        mEventHandler = null;
+    }
+
+    SRTTrack(Handler eventHandler, MediaFormat format) {
+        super(null, format);
+        mEventHandler = eventHandler;
+    }
+
+    @Override
+    protected void onData(SubtitleData data) {
+        try {
+            TextTrackCue cue = new TextTrackCue();
+            cue.mStartTimeMs = data.getStartTimeUs() / 1000;
+            cue.mEndTimeMs = (data.getStartTimeUs() + data.getDurationUs()) / 1000;
+
+            String paragraph;
+            paragraph = new String(data.getData(), "UTF-8");
+            String[] lines = paragraph.split("\\r?\\n");
+            cue.mLines = new TextTrackCueSpan[lines.length][];
+
+            int i = 0;
+            for (String line : lines) {
+                TextTrackCueSpan[] span = new TextTrackCueSpan[] {
+                    new TextTrackCueSpan(line, -1)
+                };
+                cue.mLines[i++] = span;
+            }
+
+            addCue(cue);
+        } catch (UnsupportedEncodingException e) {
+            Log.w(TAG, "subtitle data is not UTF-8 encoded: " + e);
+        }
+    }
+
+    @Override
+    public void onData(byte[] data, boolean eos, long runID) {
+        // TODO make reentrant
+        try {
+            Reader r = new InputStreamReader(new ByteArrayInputStream(data), "UTF-8");
+            BufferedReader br = new BufferedReader(r);
+
+            String header;
+            while ((header = br.readLine()) != null) {
+                // discard subtitle number
+                header  = br.readLine();
+                if (header == null) {
+                    break;
+                }
+
+                TextTrackCue cue = new TextTrackCue();
+                String[] startEnd = header.split("-->");
+                cue.mStartTimeMs = parseMs(startEnd[0]);
+                cue.mEndTimeMs = parseMs(startEnd[1]);
+
+                String s;
+                List<String> paragraph = new ArrayList<String>();
+                while (!((s = br.readLine()) == null || s.trim().equals(""))) {
+                    paragraph.add(s);
+                }
+
+                int i = 0;
+                cue.mLines = new TextTrackCueSpan[paragraph.size()][];
+                cue.mStrings = paragraph.toArray(new String[0]);
+                for (String line : paragraph) {
+                    TextTrackCueSpan[] span = new TextTrackCueSpan[] {
+                            new TextTrackCueSpan(line, -1)
+                    };
+                    cue.mStrings[i] = line;
+                    cue.mLines[i++] = span;
+                }
+
+                addCue(cue);
+            }
+
+        } catch (UnsupportedEncodingException e) {
+            Log.w(TAG, "subtitle data is not UTF-8 encoded: " + e);
+        } catch (IOException ioe) {
+            // shouldn't happen
+            Log.e(TAG, ioe.getMessage(), ioe);
+        }
+    }
+
+    @Override
+    public void updateView(Vector<Cue> activeCues) {
+        if (getRenderingWidget() != null) {
+            super.updateView(activeCues);
+            return;
+        }
+
+        if (mEventHandler == null) {
+            return;
+        }
+
+        final int _ = 0;
+        for (Cue cue : activeCues) {
+            TextTrackCue ttc = (TextTrackCue) cue;
+
+            Parcel parcel = Parcel.obtain();
+            parcel.writeInt(KEY_LOCAL_SETTING);
+            parcel.writeInt(KEY_START_TIME);
+            parcel.writeInt((int) cue.mStartTimeMs);
+
+            parcel.writeInt(KEY_STRUCT_TEXT);
+            StringBuilder sb = new StringBuilder();
+            for (String line : ttc.mStrings) {
+                sb.append(line).append('\n');
+            }
+
+            byte[] buf = sb.toString().getBytes();
+            parcel.writeInt(buf.length);
+            parcel.writeByteArray(buf);
+
+            Message msg = mEventHandler.obtainMessage(MEDIA_TIMED_TEXT, _, _, parcel);
+            mEventHandler.sendMessage(msg);
+        }
+        activeCues.clear();
+    }
+
+    private static long parseMs(String in) {
+        long hours = Long.parseLong(in.split(":")[0].trim());
+        long minutes = Long.parseLong(in.split(":")[1].trim());
+        long seconds = Long.parseLong(in.split(":")[2].split(",")[0].trim());
+        long millies = Long.parseLong(in.split(":")[2].split(",")[1].trim());
+
+        return hours * 60 * 60 * 1000 + minutes * 60 * 1000 + seconds * 1000 + millies;
+
+    }
+}
diff --git a/media/java/android/media/SubtitleController.java b/media/java/android/media/SubtitleController.java
index 13205bc..37adb8c 100644
--- a/media/java/android/media/SubtitleController.java
+++ b/media/java/android/media/SubtitleController.java
@@ -420,6 +420,19 @@ public class SubtitleController {
         }
     }
 
+    /** @hide */
+    public boolean hasRendererFor(MediaFormat format) {
+        synchronized(mRenderers) {
+            // TODO how to get available renderers in the system
+            for (Renderer renderer: mRenderers) {
+                if (renderer.supports(format)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+    }
+
     /**
      * Subtitle anchor, an object that is able to display a subtitle renderer,
      * e.g. a VideoView.
diff --git a/media/java/android/media/SubtitleTrack.java b/media/java/android/media/SubtitleTrack.java
index 9fedf63..c760810 100644
--- a/media/java/android/media/SubtitleTrack.java
+++ b/media/java/android/media/SubtitleTrack.java
@@ -274,7 +274,10 @@ public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeList
         }
 
         mVisible = true;
-        getRenderingWidget().setVisible(true);
+        RenderingWidget renderingWidget = getRenderingWidget();
+        if (renderingWidget != null) {
+            renderingWidget.setVisible(true);
+        }
         if (mTimeProvider != null) {
             mTimeProvider.scheduleUpdate(this);
         }
@@ -289,7 +292,10 @@ public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeList
         if (mTimeProvider != null) {
             mTimeProvider.cancelNotifications(this);
         }
-        getRenderingWidget().setVisible(false);
+        RenderingWidget renderingWidget = getRenderingWidget();
+        if (renderingWidget != null) {
+            renderingWidget.setVisible(false);
+        }
         mVisible = false;
     }
 
@@ -602,6 +608,12 @@ public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeList
         }
     }
 
+    /** @hide whether this is a text track who fires events instead getting rendered */
+    public boolean isTimedText() {
+        return getRenderingWidget() == null;
+    }
+
+
     /** @hide */
     private static class Run {
         public Cue mFirstCue;
diff --git a/media/java/android/media/WebVttRenderer.java b/media/java/android/media/WebVttRenderer.java
index a9374d5..69e0ea6 100644
--- a/media/java/android/media/WebVttRenderer.java
+++ b/media/java/android/media/WebVttRenderer.java
@@ -1098,7 +1098,9 @@ class WebVttTrack extends SubtitleTrack implements WebVttCueListener {
             }
         }
 
-        mRenderingWidget.setActiveCues(activeCues);
+        if (mRenderingWidget != null) {
+            mRenderingWidget.setActiveCues(activeCues);
+        }
     }
 }