Source: lib/media/segment_utils.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.media.SegmentUtils');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.log');
  11. goog.require('shaka.media.Capabilities');
  12. goog.require('shaka.media.ClosedCaptionParser');
  13. goog.require('shaka.util.BufferUtils');
  14. goog.require('shaka.util.ManifestParserUtils');
  15. goog.require('shaka.util.MimeUtils');
  16. goog.require('shaka.util.Mp4BoxParsers');
  17. goog.require('shaka.util.Mp4Parser');
  18. goog.require('shaka.util.TsParser');
  21. /**
  22.  * @summary Utility functions for segment parsing.
  23.  */
  24. shaka.media.SegmentUtils = class {
  25.   /**
  26.    * @param {string} mimeType
  27.    * @return {shaka.media.SegmentUtils.BasicInfo}
  28.    */
  29.   static getBasicInfoFromMimeType(mimeType) {
  30.     const baseMimeType = shaka.util.MimeUtils.getBasicType(mimeType);
  31.     const type = baseMimeType.split('/')[0];
  32.     const codecs = shaka.util.MimeUtils.getCodecs(mimeType);
  33.     return {
  34.       type: type,
  35.       mimeType: baseMimeType,
  36.       codecs: codecs,
  37.       language: null,
  38.       height: null,
  39.       width: null,
  40.       channelCount: null,
  41.       sampleRate: null,
  42.       closedCaptions: new Map(),
  43.       videoRange: null,
  44.       colorGamut: null,
  45.     };
  46.   }
  48.   /**
  49.    * @param {!BufferSource} data
  50.    * @return {?shaka.media.SegmentUtils.BasicInfo}
  51.    */
  52.   static getBasicInfoFromTs(data) {
  53.     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
  54.     const tsParser = new shaka.util.TsParser().parse(uint8ArrayData);
  55.     const tsCodecs = tsParser.getCodecs();
  56.     const videoInfo = tsParser.getVideoInfo();
  57.     const codecs = [];
  58.     let hasAudio = false;
  59.     let hasVideo = false;
  60.     switch (tsCodecs.audio) {
  61.       case 'aac':
  62.       case 'aac-loas':
  63.         codecs.push('mp4a.40.2');
  64.         hasAudio = true;
  65.         break;
  66.       case 'mp3':
  67.         codecs.push('mp4a.40.34');
  68.         hasAudio = true;
  69.         break;
  70.       case 'ac3':
  71.         codecs.push('ac-3');
  72.         hasAudio = true;
  73.         break;
  74.       case 'ec3':
  75.         codecs.push('ec-3');
  76.         hasAudio = true;
  77.         break;
  78.       case 'opus':
  79.         codecs.push('opus');
  80.         hasAudio = true;
  81.         break;
  82.     }
  83.     switch (tsCodecs.video) {
  84.       case 'avc':
  85.         if (videoInfo.codec) {
  86.           codecs.push(videoInfo.codec);
  87.         } else {
  88.           codecs.push('avc1.42E01E');
  89.         }
  90.         hasVideo = true;
  91.         break;
  92.       case 'hvc':
  93.         if (videoInfo.codec) {
  94.           codecs.push(videoInfo.codec);
  95.         } else {
  96.           codecs.push('hvc1.1.6.L93.90');
  97.         }
  98.         hasVideo = true;
  99.         break;
  100.       case 'av1':
  101.         codecs.push('av01.0.01M.08');
  102.         hasVideo = true;
  103.         break;
  104.     }
  105.     if (!codecs.length) {
  106.       return null;
  107.     }
  108.     const onlyAudio = hasAudio && !hasVideo;
  109.     const closedCaptions = new Map();
  110.     if (hasVideo) {
  111.       const captionParser = new shaka.media.ClosedCaptionParser('video/mp2t');
  112.       captionParser.parseFrom(data);
  113.       for (const stream of captionParser.getStreams()) {
  114.         closedCaptions.set(stream, stream);
  115.       }
  116.       captionParser.reset();
  117.     }
  118.     return {
  119.       type: onlyAudio ? 'audio' : 'video',
  120.       mimeType: 'video/mp2t',
  121.       codecs: codecs.join(', '),
  122.       language: null,
  123.       height: videoInfo.height,
  124.       width: videoInfo.width,
  125.       channelCount: null,
  126.       sampleRate: null,
  127.       closedCaptions: closedCaptions,
  128.       videoRange: null,
  129.       colorGamut: null,
  130.     };
  131.   }
  133.   /**
  134.    * @param {?BufferSource} initData
  135.    * @param {!BufferSource} data
  136.    * @return {?shaka.media.SegmentUtils.BasicInfo}
  137.    */
  138.   static getBasicInfoFromMp4(initData, data) {
  139.     const Mp4Parser = shaka.util.Mp4Parser;
  140.     const SegmentUtils = shaka.media.SegmentUtils;
  142.     const audioCodecs = [];
  143.     const videoCodecs = [];
  145.     let hasAudio = false;
  146.     let hasVideo = false;
  148.     const addCodec = (codec) => {
  149.       const codecLC = codec.toLowerCase();
  150.       switch (codecLC) {
  151.         case 'avc1':
  152.         case 'avc3':
  153.           videoCodecs.push(codecLC + '.42E01E');
  154.           hasVideo = true;
  155.           break;
  156.         case 'hev1':
  157.         case 'hvc1':
  158.           videoCodecs.push(codecLC + '.1.6.L93.90');
  159.           hasVideo = true;
  160.           break;
  161.         case 'dvh1':
  162.         case 'dvhe':
  163.           videoCodecs.push(codecLC + '.05.04');
  164.           hasVideo = true;
  165.           break;
  166.         case 'vp09':
  167.           videoCodecs.push(codecLC + '.00.10.08');
  168.           hasVideo = true;
  169.           break;
  170.         case 'av01':
  171.           videoCodecs.push(codecLC + '.0.01M.08');
  172.           hasVideo = true;
  173.           break;
  174.         case 'mp4a':
  175.           // We assume AAC, but this can be wrong since mp4a supports
  176.           // others codecs
  177.           audioCodecs.push('mp4a.40.2');
  178.           hasAudio = true;
  179.           break;
  180.         case 'ac-3':
  181.         case 'ec-3':
  182.         case 'opus':
  183.         case 'flac':
  184.           audioCodecs.push(codecLC);
  185.           hasAudio = true;
  186.           break;
  187.       }
  188.     };
  190.     const codecBoxParser = (box) => addCodec(box.name);
  192.     /** @type {?string} */
  193.     let language = null;
  194.     /** @type {?string} */
  195.     let height = null;
  196.     /** @type {?string} */
  197.     let width = null;
  198.     /** @type {?number} */
  199.     let channelCount = null;
  200.     /** @type {?number} */
  201.     let sampleRate = null;
  202.     /** @type {?string} */
  203.     let realVideoRange = null;
  204.     /** @type {?string} */
  205.     let realColorGamut = null;
  207.     /** @type {?string} */
  208.     let baseBox;
  210.     new Mp4Parser()
  211.         .box('moov', Mp4Parser.children)
  212.         .box('trak', Mp4Parser.children)
  213.         .fullBox('tkhd', (box) => {
  214.           goog.asserts.assert(
  215.               box.version != null,
  216.               'TKHD is a full box and should have a valid version.');
  217.           const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
  218.               box.reader, box.version);
  219.           height = String(parsedTKHDBox.height);
  220.           width = String(parsedTKHDBox.width);
  221.         })
  222.         .box('mdia', Mp4Parser.children)
  223.         .fullBox('mdhd', (box) => {
  224.           goog.asserts.assert(
  225.               box.version != null,
  226.               'MDHD is a full box and should have a valid version.');
  227.           const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  228.               box.reader, box.version);
  229.           language = parsedMDHDBox.language;
  230.         })
  231.         .box('minf', Mp4Parser.children)
  232.         .box('stbl', Mp4Parser.children)
  233.         .fullBox('stsd', Mp4Parser.sampleDescription)
  235.         // AUDIO
  236.         // These are the various boxes that signal a codec.
  237.         .box('mp4a', (box) => {
  238.           const parsedMP4ABox = shaka.util.Mp4BoxParsers.parseMP4A(box.reader);
  239.           channelCount = parsedMP4ABox.channelCount;
  240.           sampleRate = parsedMP4ABox.sampleRate;
  241.           if (box.reader.hasMoreData()) {
  242.             Mp4Parser.children(box);
  243.           } else {
  244.             codecBoxParser(box);
  245.           }
  246.         })
  247.         .box('esds', (box) => {
  248.           const parsedESDSBox = shaka.util.Mp4BoxParsers.parseESDS(box.reader);
  249.           audioCodecs.push(parsedESDSBox.codec);
  250.           hasAudio = true;
  251.         })
  252.         .box('ac-3', codecBoxParser)
  253.         .box('ec-3', codecBoxParser)
  254.         .box('opus', codecBoxParser)
  255.         .box('Opus', codecBoxParser)
  256.         .box('fLaC', codecBoxParser)
  258.         // VIDEO
  259.         // These are the various boxes that signal a codec.
  260.         .box('avc1', (box) => {
  261.           baseBox = box.name;
  262.           Mp4Parser.visualSampleEntry(box);
  263.         })
  264.         .box('avc3', (box) => {
  265.           baseBox = box.name;
  266.           Mp4Parser.visualSampleEntry(box);
  267.         })
  268.         .box('hev1', (box) => {
  269.           baseBox = box.name;
  270.           Mp4Parser.visualSampleEntry(box);
  271.         })
  272.         .box('hvc1', (box) => {
  273.           baseBox = box.name;
  274.           Mp4Parser.visualSampleEntry(box);
  275.         })
  276.         .box('dva1', (box) => {
  277.           baseBox = box.name;
  278.           Mp4Parser.visualSampleEntry(box);
  279.         })
  280.         .box('dvav', (box) => {
  281.           baseBox = box.name;
  282.           Mp4Parser.visualSampleEntry(box);
  283.         })
  284.         .box('dvh1', (box) => {
  285.           baseBox = box.name;
  286.           Mp4Parser.visualSampleEntry(box);
  287.         })
  288.         .box('dvhe', (box) => {
  289.           baseBox = box.name;
  290.           Mp4Parser.visualSampleEntry(box);
  291.         })
  292.         .box('vp09', (box) => {
  293.           baseBox = box.name;
  294.           Mp4Parser.visualSampleEntry(box);
  295.         })
  296.         .box('av01', (box) => {
  297.           baseBox = box.name;
  298.           Mp4Parser.visualSampleEntry(box);
  299.         })
  300.         .box('avcC', (box) => {
  301.           let codecBase = baseBox || '';
  302.           switch (baseBox) {
  303.             case 'dvav':
  304.               codecBase = 'avc3';
  305.               break;
  306.             case 'dva1':
  307.               codecBase = 'avc1';
  308.               break;
  309.           }
  310.           const parsedAVCCBox = shaka.util.Mp4BoxParsers.parseAVCC(
  311.               codecBase, box.reader, box.name);
  312.           videoCodecs.push(parsedAVCCBox.codec);
  313.           hasVideo = true;
  314.         })
  315.         .box('hvcC', (box) => {
  316.           let codecBase = baseBox || '';
  317.           switch (baseBox) {
  318.             case 'dvh1':
  319.               codecBase = 'hvc1';
  320.               break;
  321.             case 'dvhe':
  322.               codecBase = 'hev1';
  323.               break;
  324.           }
  325.           const parsedHVCCBox = shaka.util.Mp4BoxParsers.parseHVCC(
  326.               codecBase, box.reader, box.name);
  327.           videoCodecs.push(parsedHVCCBox.codec);
  328.           hasVideo = true;
  329.         })
  330.         .box('dvcC', (box) => {
  331.           let codecBase = baseBox || '';
  332.           switch (baseBox) {
  333.             case 'hvc1':
  334.               codecBase = 'dvh1';
  335.               break;
  336.             case 'hev1':
  337.               codecBase = 'dvhe';
  338.               break;
  339.             case 'avc1':
  340.               codecBase = 'dva1';
  341.               break;
  342.             case 'avc3':
  343.               codecBase = 'dvav';
  344.               break;
  345.           }
  346.           const parsedDVCCBox = shaka.util.Mp4BoxParsers.parseDVCC(
  347.               codecBase, box.reader, box.name);
  348.           videoCodecs.push(parsedDVCCBox.codec);
  349.           hasVideo = true;
  350.         })
  351.         .fullBox('vpcC', (box) => {
  352.           const codecBase = baseBox || '';
  353.           const parsedVPCCBox = shaka.util.Mp4BoxParsers.parseVPCC(
  354.               codecBase, box.reader, box.name);
  355.           videoCodecs.push(parsedVPCCBox.codec);
  356.           hasVideo = true;
  357.         })
  358.         .box('av1C', (box) => {
  359.           const codecBase = baseBox || '';
  360.           const parsedAV1CBox = shaka.util.Mp4BoxParsers.parseAV1C(
  361.               codecBase, box.reader, box.name);
  362.           videoCodecs.push(parsedAV1CBox.codec);
  363.           hasVideo = true;
  364.         })
  366.         // This signals an encrypted sample, which we can go inside of to
  367.         // find the codec used.
  368.         // Note: If encrypted, you can only have audio or video, not both.
  369.         .box('enca', Mp4Parser.audioSampleEntry)
  370.         .box('encv', Mp4Parser.visualSampleEntry)
  371.         .box('sinf', Mp4Parser.children)
  372.         .box('frma', (box) => {
  373.           const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
  374.           addCodec(codec);
  375.         })
  377.         .box('colr', (box) => {
  378.           const {videoRange, colorGamut} =
  379.               shaka.util.Mp4BoxParsers.parseCOLR(box.reader);
  380.           realVideoRange = videoRange;
  381.           realColorGamut = colorGamut;
  382.         })
  384.         .parse(initData || data, /* partialOkay= */ true);
  385.     if (!audioCodecs.length && !videoCodecs.length) {
  386.       return null;
  387.     }
  388.     const onlyAudio = hasAudio && !hasVideo;
  389.     const closedCaptions = new Map();
  390.     if (hasVideo) {
  391.       const captionParser = new shaka.media.ClosedCaptionParser('video/mp4');
  392.       if (initData) {
  393.         captionParser.init(initData);
  394.       }
  395.       captionParser.parseFrom(data);
  396.       for (const stream of captionParser.getStreams()) {
  397.         closedCaptions.set(stream, stream);
  398.       }
  399.       captionParser.reset();
  400.     }
  401.     const codecs = audioCodecs.concat(videoCodecs);
  402.     return {
  403.       type: onlyAudio ? 'audio' : 'video',
  404.       mimeType: onlyAudio ? 'audio/mp4' : 'video/mp4',
  405.       codecs: SegmentUtils.codecsFiltering(codecs).join(', '),
  406.       language: language,
  407.       height: height,
  408.       width: width,
  409.       channelCount: channelCount,
  410.       sampleRate: sampleRate,
  411.       closedCaptions: closedCaptions,
  412.       videoRange: realVideoRange,
  413.       colorGamut: realColorGamut,
  414.     };
  415.   }
  417.   /**
  418.    * @param {!Array.<string>} codecs
  419.    * @return {!Array.<string>} codecs
  420.    */
  421.   static codecsFiltering(codecs) {
  422.     const ContentType = shaka.util.ManifestParserUtils.ContentType;
  423.     const ManifestParserUtils = shaka.util.ManifestParserUtils;
  424.     const SegmentUtils = shaka.media.SegmentUtils;
  425.     const allCodecs = SegmentUtils.filterDuplicateCodecs_(codecs);
  426.     const audioCodecs =
  427.         ManifestParserUtils.guessAllCodecsSafe(ContentType.AUDIO, allCodecs);
  428.     const videoCodecs =
  429.         ManifestParserUtils.guessAllCodecsSafe(ContentType.VIDEO, allCodecs);
  430.     const textCodecs =
  431.         ManifestParserUtils.guessAllCodecsSafe(ContentType.TEXT, allCodecs);
  432.     const validVideoCodecs = SegmentUtils.chooseBetterCodecs_(videoCodecs);
  433.     const finalCodecs =
  434.         audioCodecs.concat(validVideoCodecs).concat(textCodecs);
  435.     if (allCodecs.length && !finalCodecs.length) {
  436.       return allCodecs;
  437.     }
  438.     return finalCodecs;
  439.   }
  441.   /**
  442.    * @param {!Array.<string>} codecs
  443.    * @return {!Array.<string>} codecs
  444.    * @private
  445.    */
  446.   static filterDuplicateCodecs_(codecs) {
  447.     // Filter out duplicate codecs.
  448.     const seen = new Set();
  449.     const ret = [];
  450.     for (const codec of codecs) {
  451.       const shortCodec = shaka.util.MimeUtils.getCodecBase(codec);
  452.       if (!seen.has(shortCodec)) {
  453.         ret.push(codec);
  454.         seen.add(shortCodec);
  455.       } else {
  456.         shaka.log.debug('Ignoring duplicate codec');
  457.       }
  458.     }
  459.     return ret;
  460.   }
  462.   /**
  463.    * Prioritizes Dolby Vision if supported. This is necessary because with
  464.    * Dolby Vision we could have hvcC and dvcC boxes at the same time.
  465.    *
  466.    * @param {!Array.<string>} codecs
  467.    * @return {!Array.<string>} codecs
  468.    * @private
  469.    */
  470.   static chooseBetterCodecs_(codecs) {
  471.     if (codecs.length <= 1) {
  472.       return codecs;
  473.     }
  474.     const dolbyVision = codecs.find((codec) => {
  475.       return codec.startsWith('dvh1.') ||
  476.           codec.startsWith('dvhe.') ||
  477.           codec.startsWith('dav1.');
  478.     });
  479.     if (!dolbyVision) {
  480.       return codecs;
  481.     }
  482.     const type = `video/mp4; codecs="${dolbyVision}"`;
  483.     if (shaka.media.Capabilities.isTypeSupported(type)) {
  484.       return [dolbyVision];
  485.     }
  486.     return codecs.filter((codec) => codec != dolbyVision);
  487.   }
  489.   /**
  490.    * @param {!BufferSource} data
  491.    * @return {?string}
  492.    */
  493.   static getDefaultKID(data) {
  494.     const Mp4Parser = shaka.util.Mp4Parser;
  496.     let defaultKID = null;
  497.     new Mp4Parser()
  498.         .box('moov', Mp4Parser.children)
  499.         .box('trak', Mp4Parser.children)
  500.         .box('mdia', Mp4Parser.children)
  501.         .box('minf', Mp4Parser.children)
  502.         .box('stbl', Mp4Parser.children)
  503.         .fullBox('stsd', Mp4Parser.sampleDescription)
  504.         .box('encv', Mp4Parser.visualSampleEntry)
  505.         .box('enca', Mp4Parser.audioSampleEntry)
  506.         .box('sinf', Mp4Parser.children)
  507.         .box('schi', Mp4Parser.children)
  508.         .fullBox('tenc', (box) => {
  509.           const parsedTENCBox = shaka.util.Mp4BoxParsers.parseTENC(box.reader);
  510.           defaultKID = parsedTENCBox.defaultKID;
  511.         })
  513.         .parse(data, /* partialOkay= */ true);
  514.     return defaultKID;
  515.   }
  516. };
  519. /**
  520.  * @typedef {{
  521.  *   type: string,
  522.  *   mimeType: string,
  523.  *   codecs: string,
  524.  *   language: ?string,
  525.  *   height: ?string,
  526.  *   width: ?string,
  527.  *   channelCount: ?number,
  528.  *   sampleRate: ?number,
  529.  *   closedCaptions: Map.<string, string>,
  530.  *   videoRange: ?string,
  531.  *   colorGamut: ?string
  532.  * }}
  533.  *
  534.  * @property {string} type
  535.  * @property {string} mimeType
  536.  * @property {string} codecs
  537.  * @property {?string} language
  538.  * @property {?string} height
  539.  * @property {?string} width
  540.  * @property {?number} channelCount
  541.  * @property {?number} sampleRate
  542.  * @property {Map.<string, string>} closedCaptions
  543.  * @property {?string} videoRange
  544.  * @property {?string} colorGamut
  545.  */
  546. shaka.media.SegmentUtils.BasicInfo;