{"data":{"id":"10.48550/arxiv.2006.16668","type":"dois","attributes":{"doi":"10.48550/arxiv.2006.16668","prefix":"10.48550","suffix":"arxiv.2006.16668","identifiers":[{"identifier":"2006.16668","identifierType":"arXiv"}],"alternateIdentifiers":[{"alternateIdentifierType":"arXiv","alternateIdentifier":"2006.16668"}],"creators":[{"name":"Lepikhin, Dmitry","nameType":"Personal","givenName":"Dmitry","familyName":"Lepikhin","affiliation":[],"nameIdentifiers":[]},{"name":"Lee, HyoukJoong","nameType":"Personal","givenName":"HyoukJoong","familyName":"Lee","affiliation":[],"nameIdentifiers":[]},{"name":"Xu, Yuanzhong","nameType":"Personal","givenName":"Yuanzhong","familyName":"Xu","affiliation":[],"nameIdentifiers":[]},{"name":"Chen, Dehao","nameType":"Personal","givenName":"Dehao","familyName":"Chen","affiliation":[],"nameIdentifiers":[]},{"name":"Firat, Orhan","nameType":"Personal","givenName":"Orhan","familyName":"Firat","affiliation":[],"nameIdentifiers":[]},{"name":"Huang, Yanping","nameType":"Personal","givenName":"Yanping","familyName":"Huang","affiliation":[],"nameIdentifiers":[]},{"name":"Krikun, Maxim","nameType":"Personal","givenName":"Maxim","familyName":"Krikun","affiliation":[],"nameIdentifiers":[]},{"name":"Shazeer, Noam","nameType":"Personal","givenName":"Noam","familyName":"Shazeer","affiliation":[],"nameIdentifiers":[]},{"name":"Chen, Zhifeng","nameType":"Personal","givenName":"Zhifeng","familyName":"Chen","affiliation":[],"nameIdentifiers":[]}],"titles":[{"title":"GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding"}],"publisher":"arXiv","container":{},"publicationYear":2020,"subjects":[{"lang":"en","subject":"Computation and Language (cs.CL)","subjectScheme":"arXiv"},{"lang":"en","subject":"Machine Learning (cs.LG)","subjectScheme":"arXiv"},{"lang":"en","subject":"Machine Learning (stat.ML)","subjectScheme":"arXiv"},{"subject":"FOS: Computer and information sciences","subjectScheme":"Fields of Science and Technology (FOS)"},{"subject":"FOS: Computer and information sciences","schemeUri":"http://www.oecd.org/science/inno/38235147.pdf","subjectScheme":"Fields of Science and Technology (FOS)"}],"contributors":[],"dates":[{"date":"2020-06-30T10:42:02Z","dateType":"Submitted","dateInformation":"v1"},{"date":"2020-07-01T00:16:23Z","dateType":"Updated","dateInformation":"v1"},{"date":"2020-06","dateType":"Available","dateInformation":"v1"},{"date":"2020","dateType":"Issued"}],"language":null,"types":{"ris":"GEN","bibtex":"misc","citeproc":"article","schemaOrg":"CreativeWork","resourceType":"Article","resourceTypeGeneral":"Preprint"},"relatedIdentifiers":[],"relatedItems":[],"sizes":[],"formats":[],"version":"1","rightsList":[{"rights":"arXiv.org perpetual, non-exclusive license","rightsUri":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/"}],"descriptions":[{"description":"Neural network scaling has been critical for improving the model quality in many real-world machine learning applications with vast amounts of training data and compute. Although this trend of scaling is affirmed to be a sure-fire approach for better model quality, there are challenges on the path such as the computation cost, ease of programming, and efficient implementation on parallel devices. GShard is a module composed of a set of lightweight annotation APIs and an extension to the XLA compiler. It provides an elegant way to express a wide range of parallel computation patterns with minimal changes to the existing model code. GShard enabled us to scale up multilingual neural machine translation Transformer model with Sparsely-Gated Mixture-of-Experts beyond 600 billion parameters using automatic sharding. We demonstrate that such a giant model can efficiently be trained on 2048 TPU v3 accelerators in 4 days to achieve far superior quality for translation from 100 languages to English compared to the prior art.","descriptionType":"Abstract"}],"geoLocations":[],"fundingReferences":[],"xml":"PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPHJlc291cmNlIHhtbG5zPSJodHRwOi8vZGF0YWNpdGUub3JnL3NjaGVtYS9rZXJuZWwtNCIgeG1sbnM6eHNpPSJodHRwOi8vd3d3LnczLm9yZy8yMDAxL1hNTFNjaGVtYS1pbnN0YW5jZSIgeHNpOnNjaGVtYUxvY2F0aW9uPSJodHRwOi8vZGF0YWNpdGUub3JnL3NjaGVtYS9rZXJuZWwtNCBodHRwOi8vc2NoZW1hLmRhdGFjaXRlLm9yZy9tZXRhL2tlcm5lbC00LjMvbWV0YWRhdGEueHNkIj4KICA8aWRlbnRpZmllciBpZGVudGlmaWVyVHlwZT0iRE9JIj4xMC40ODU1MC9BUlhJVi4yMDA2LjE2NjY4PC9pZGVudGlmaWVyPgogIDxhbHRlcm5hdGVJZGVudGlmaWVycz4KICAgIDxhbHRlcm5hdGVJZGVudGlmaWVyIGFsdGVybmF0ZUlkZW50aWZpZXJUeXBlPSJhclhpdiI+MjAwNi4xNjY2ODwvYWx0ZXJuYXRlSWRlbnRpZmllcj4KICA8L2FsdGVybmF0ZUlkZW50aWZpZXJzPgogIDxjcmVhdG9ycz4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5MZXBpa2hpbiwgRG1pdHJ5PC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5EbWl0cnk8L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+TGVwaWtoaW48L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+TGVlLCBIeW91a0pvb25nPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5IeW91a0pvb25nPC9naXZlbk5hbWU+CiAgICAgIDxmYW1pbHlOYW1lPkxlZTwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5YdSwgWXVhbnpob25nPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5ZdWFuemhvbmc8L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+WHU8L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+Q2hlbiwgRGVoYW88L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPkRlaGFvPC9naXZlbk5hbWU+CiAgICAgIDxmYW1pbHlOYW1lPkNoZW48L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+RmlyYXQsIE9yaGFuPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5PcmhhbjwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5GaXJhdDwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5IdWFuZywgWWFucGluZzwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+WWFucGluZzwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5IdWFuZzwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5LcmlrdW4sIE1heGltPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5NYXhpbTwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5LcmlrdW48L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+U2hhemVlciwgTm9hbTwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+Tm9hbTwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5TaGF6ZWVyPC9mYW1pbHlOYW1lPgogICAgPC9jcmVhdG9yPgogICAgPGNyZWF0b3I+CiAgICAgIDxjcmVhdG9yTmFtZSBuYW1lVHlwZT0iUGVyc29uYWwiPkNoZW4sIFpoaWZlbmc8L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPlpoaWZlbmc8L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+Q2hlbjwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICA8L2NyZWF0b3JzPgogIDx0aXRsZXM+CiAgICA8dGl0bGU+R1NoYXJkOiBTY2FsaW5nIEdpYW50IE1vZGVscyB3aXRoIENvbmRpdGlvbmFsIENvbXB1dGF0aW9uIGFuZCBBdXRvbWF0aWMgU2hhcmRpbmc8L3RpdGxlPgogIDwvdGl0bGVzPgogIDxwdWJsaXNoZXI+YXJYaXY8L3B1Ymxpc2hlcj4KICA8cHVibGljYXRpb25ZZWFyPjIwMjA8L3B1YmxpY2F0aW9uWWVhcj4KICA8c3ViamVjdHM+CiAgICA8c3ViamVjdCB4bWw6bGFuZz0iZW4iIHN1YmplY3RTY2hlbWU9ImFyWGl2Ij5Db21wdXRhdGlvbiBhbmQgTGFuZ3VhZ2UgKGNzLkNMKTwvc3ViamVjdD4KICAgIDxzdWJqZWN0IHhtbDpsYW5nPSJlbiIgc3ViamVjdFNjaGVtZT0iYXJYaXYiPk1hY2hpbmUgTGVhcm5pbmcgKGNzLkxHKTwvc3ViamVjdD4KICAgIDxzdWJqZWN0IHhtbDpsYW5nPSJlbiIgc3ViamVjdFNjaGVtZT0iYXJYaXYiPk1hY2hpbmUgTGVhcm5pbmcgKHN0YXQuTUwpPC9zdWJqZWN0PgogICAgPHN1YmplY3Qgc3ViamVjdFNjaGVtZT0iRmllbGRzIG9mIFNjaWVuY2UgYW5kIFRlY2hub2xvZ3kgKEZPUykiPkZPUzogQ29tcHV0ZXIgYW5kIGluZm9ybWF0aW9uIHNjaWVuY2VzPC9zdWJqZWN0PgogIDwvc3ViamVjdHM+CiAgPGRhdGVzPgogICAgPGRhdGUgZGF0ZVR5cGU9IlN1Ym1pdHRlZCIgZGF0ZUluZm9ybWF0aW9uPSJ2MSI+MjAyMC0wNi0zMFQxMDo0MjowMlo8L2RhdGU+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iVXBkYXRlZCIgZGF0ZUluZm9ybWF0aW9uPSJ2MSI+MjAyMC0wNy0wMVQwMDoxNjoyM1o8L2RhdGU+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iQXZhaWxhYmxlIiBkYXRlSW5mb3JtYXRpb249InYxIj4yMDIwLTA2PC9kYXRlPgogIDwvZGF0ZXM+CiAgPHJlc291cmNlVHlwZSByZXNvdXJjZVR5cGVHZW5lcmFsPSJQcmVwcmludCI+QXJ0aWNsZTwvcmVzb3VyY2VUeXBlPgogIDx2ZXJzaW9uPjE8L3ZlcnNpb24+CiAgPHJpZ2h0c0xpc3Q+CiAgICA8cmlnaHRzIHJpZ2h0c1VSST0iaHR0cDovL2FyeGl2Lm9yZy9saWNlbnNlcy9ub25leGNsdXNpdmUtZGlzdHJpYi8xLjAvIj5hclhpdi5vcmcgcGVycGV0dWFsLCBub24tZXhjbHVzaXZlIGxpY2Vuc2U8L3JpZ2h0cz4KICA8L3JpZ2h0c0xpc3Q+CiAgPGRlc2NyaXB0aW9ucz4KICAgIDxkZXNjcmlwdGlvbiBkZXNjcmlwdGlvblR5cGU9IkFic3RyYWN0Ij5OZXVyYWwgbmV0d29yayBzY2FsaW5nIGhhcyBiZWVuIGNyaXRpY2FsIGZvciBpbXByb3ZpbmcgdGhlIG1vZGVsIHF1YWxpdHkgaW4gbWFueSByZWFsLXdvcmxkIG1hY2hpbmUgbGVhcm5pbmcgYXBwbGljYXRpb25zIHdpdGggdmFzdCBhbW91bnRzIG9mIHRyYWluaW5nIGRhdGEgYW5kIGNvbXB1dGUuIEFsdGhvdWdoIHRoaXMgdHJlbmQgb2Ygc2NhbGluZyBpcyBhZmZpcm1lZCB0byBiZSBhIHN1cmUtZmlyZSBhcHByb2FjaCBmb3IgYmV0dGVyIG1vZGVsIHF1YWxpdHksIHRoZXJlIGFyZSBjaGFsbGVuZ2VzIG9uIHRoZSBwYXRoIHN1Y2ggYXMgdGhlIGNvbXB1dGF0aW9uIGNvc3QsIGVhc2Ugb2YgcHJvZ3JhbW1pbmcsIGFuZCBlZmZpY2llbnQgaW1wbGVtZW50YXRpb24gb24gcGFyYWxsZWwgZGV2aWNlcy4gR1NoYXJkIGlzIGEgbW9kdWxlIGNvbXBvc2VkIG9mIGEgc2V0IG9mIGxpZ2h0d2VpZ2h0IGFubm90YXRpb24gQVBJcyBhbmQgYW4gZXh0ZW5zaW9uIHRvIHRoZSBYTEEgY29tcGlsZXIuIEl0IHByb3ZpZGVzIGFuIGVsZWdhbnQgd2F5IHRvIGV4cHJlc3MgYSB3aWRlIHJhbmdlIG9mIHBhcmFsbGVsIGNvbXB1dGF0aW9uIHBhdHRlcm5zIHdpdGggbWluaW1hbCBjaGFuZ2VzIHRvIHRoZSBleGlzdGluZyBtb2RlbCBjb2RlLiBHU2hhcmQgZW5hYmxlZCB1cyB0byBzY2FsZSB1cCBtdWx0aWxpbmd1YWwgbmV1cmFsIG1hY2hpbmUgdHJhbnNsYXRpb24gVHJhbnNmb3JtZXIgbW9kZWwgd2l0aCBTcGFyc2VseS1HYXRlZCBNaXh0dXJlLW9mLUV4cGVydHMgYmV5b25kIDYwMCBiaWxsaW9uIHBhcmFtZXRlcnMgdXNpbmcgYXV0b21hdGljIHNoYXJkaW5nLiBXZSBkZW1vbnN0cmF0ZSB0aGF0IHN1Y2ggYSBnaWFudCBtb2RlbCBjYW4gZWZmaWNpZW50bHkgYmUgdHJhaW5lZCBvbiAyMDQ4IFRQVSB2MyBhY2NlbGVyYXRvcnMgaW4gNCBkYXlzIHRvIGFjaGlldmUgZmFyIHN1cGVyaW9yIHF1YWxpdHkgZm9yIHRyYW5zbGF0aW9uIGZyb20gMTAwIGxhbmd1YWdlcyB0byBFbmdsaXNoIGNvbXBhcmVkIHRvIHRoZSBwcmlvciBhcnQuPC9kZXNjcmlwdGlvbj4KICA8L2Rlc2NyaXB0aW9ucz4KPC9yZXNvdXJjZT4=","url":"https://arxiv.org/abs/2006.16668","contentUrl":null,"metadataVersion":0,"schemaVersion":"http://datacite.org/schema/kernel-4","source":"mds","isActive":true,"state":"findable","reason":null,"viewCount":0,"viewsOverTime":[],"downloadCount":0,"downloadsOverTime":[],"referenceCount":0,"citationCount":0,"citationsOverTime":[],"partCount":0,"partOfCount":0,"versionCount":0,"versionOfCount":0,"created":"2022-02-25T03:03:56.000Z","registered":"2022-02-25T03:03:57.000Z","published":"2020","updated":"2022-02-25T03:03:57.000Z"},"relationships":{"client":{"data":{"id":"arxiv.content","type":"clients"}},"provider":{"data":{"id":"arxiv","type":"providers"}},"media":{"data":{"id":"10.48550/arxiv.2006.16668","type":"media"}},"references":{"data":[]},"citations":{"data":[]},"parts":{"data":[]},"partOf":{"data":[]},"versions":{"data":[]},"versionOf":{"data":[]}}}}