{"data":{"id":"10.48550/arxiv.1910.10683","type":"dois","attributes":{"doi":"10.48550/arxiv.1910.10683","prefix":"10.48550","suffix":"arxiv.1910.10683","identifiers":[{"identifier":"1910.10683","identifierType":"arXiv"}],"alternateIdentifiers":[{"alternateIdentifierType":"arXiv","alternateIdentifier":"1910.10683"}],"creators":[{"name":"Raffel, Colin","nameType":"Personal","givenName":"Colin","familyName":"Raffel","affiliation":[],"nameIdentifiers":[]},{"name":"Shazeer, Noam","nameType":"Personal","givenName":"Noam","familyName":"Shazeer","affiliation":[],"nameIdentifiers":[]},{"name":"Roberts, Adam","nameType":"Personal","givenName":"Adam","familyName":"Roberts","affiliation":[],"nameIdentifiers":[]},{"name":"Lee, Katherine","nameType":"Personal","givenName":"Katherine","familyName":"Lee","affiliation":[],"nameIdentifiers":[]},{"name":"Narang, Sharan","nameType":"Personal","givenName":"Sharan","familyName":"Narang","affiliation":[],"nameIdentifiers":[]},{"name":"Matena, Michael","nameType":"Personal","givenName":"Michael","familyName":"Matena","affiliation":[],"nameIdentifiers":[]},{"name":"Zhou, Yanqi","nameType":"Personal","givenName":"Yanqi","familyName":"Zhou","affiliation":[],"nameIdentifiers":[]},{"name":"Li, Wei","nameType":"Personal","givenName":"Wei","familyName":"Li","affiliation":[],"nameIdentifiers":[]},{"name":"Liu, Peter J.","nameType":"Personal","givenName":"Peter J.","familyName":"Liu","affiliation":[],"nameIdentifiers":[]}],"titles":[{"title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer"}],"publisher":"arXiv","container":{},"publicationYear":2019,"subjects":[{"lang":"en","subject":"Machine Learning (cs.LG)","subjectScheme":"arXiv"},{"lang":"en","subject":"Computation and Language (cs.CL)","subjectScheme":"arXiv"},{"lang":"en","subject":"Machine Learning (stat.ML)","subjectScheme":"arXiv"},{"subject":"FOS: Computer and information sciences","subjectScheme":"Fields of Science and Technology (FOS)"},{"subject":"FOS: Computer and information sciences","schemeUri":"http://www.oecd.org/science/inno/38235147.pdf","subjectScheme":"Fields of Science and Technology (FOS)"}],"contributors":[],"dates":[{"date":"2019-10-23T17:37:36Z","dateType":"Submitted","dateInformation":"v1"},{"date":"2019-10-24T00:17:18Z","dateType":"Updated","dateInformation":"v1"},{"date":"2019-10-24T15:13:50Z","dateType":"Submitted","dateInformation":"v2"},{"date":"2019-10-25T00:18:00Z","dateType":"Updated","dateInformation":"v2"},{"date":"2020-07-28T13:10:01Z","dateType":"Submitted","dateInformation":"v3"},{"date":"2020-07-29T00:16:56Z","dateType":"Updated","dateInformation":"v3"},{"date":"2023-09-19T15:14:48Z","dateType":"Submitted","dateInformation":"v4"},{"date":"2023-09-20T00:20:34Z","dateType":"Updated","dateInformation":"v4"},{"date":"2019-10","dateType":"Available","dateInformation":"v1"},{"date":"2019","dateType":"Issued"}],"language":null,"types":{"ris":"GEN","bibtex":"misc","citeproc":"article","schemaOrg":"CreativeWork","resourceType":"Article","resourceTypeGeneral":"Preprint"},"relatedIdentifiers":[],"relatedItems":[],"sizes":[],"formats":[],"version":"4","rightsList":[{"rights":"arXiv.org perpetual, non-exclusive license","rightsUri":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/"}],"descriptions":[{"description":"Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts all text-based language problems into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled data sets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new ``Colossal Clean Crawled Corpus'', we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our data set, pre-trained models, and code.","descriptionType":"Abstract"}],"geoLocations":[],"fundingReferences":[],"xml":"PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPHJlc291cmNlIHhtbG5zPSJodHRwOi8vZGF0YWNpdGUub3JnL3NjaGVtYS9rZXJuZWwtNCIgeG1sbnM6eHNpPSJodHRwOi8vd3d3LnczLm9yZy8yMDAxL1hNTFNjaGVtYS1pbnN0YW5jZSIgeHNpOnNjaGVtYUxvY2F0aW9uPSJodHRwOi8vZGF0YWNpdGUub3JnL3NjaGVtYS9rZXJuZWwtNCBodHRwOi8vc2NoZW1hLmRhdGFjaXRlLm9yZy9tZXRhL2tlcm5lbC00LjMvbWV0YWRhdGEueHNkIj4KICA8aWRlbnRpZmllciBpZGVudGlmaWVyVHlwZT0iRE9JIj4xMC40ODU1MC9BUlhJVi4xOTEwLjEwNjgzPC9pZGVudGlmaWVyPgogIDxhbHRlcm5hdGVJZGVudGlmaWVycz4KICAgIDxhbHRlcm5hdGVJZGVudGlmaWVyIGFsdGVybmF0ZUlkZW50aWZpZXJUeXBlPSJhclhpdiI+MTkxMC4xMDY4MzwvYWx0ZXJuYXRlSWRlbnRpZmllcj4KICA8L2FsdGVybmF0ZUlkZW50aWZpZXJzPgogIDxjcmVhdG9ycz4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5SYWZmZWwsIENvbGluPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5Db2xpbjwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5SYWZmZWw8L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+U2hhemVlciwgTm9hbTwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+Tm9hbTwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5TaGF6ZWVyPC9mYW1pbHlOYW1lPgogICAgPC9jcmVhdG9yPgogICAgPGNyZWF0b3I+CiAgICAgIDxjcmVhdG9yTmFtZSBuYW1lVHlwZT0iUGVyc29uYWwiPlJvYmVydHMsIEFkYW08L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPkFkYW08L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+Um9iZXJ0czwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5MZWUsIEthdGhlcmluZTwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+S2F0aGVyaW5lPC9naXZlbk5hbWU+CiAgICAgIDxmYW1pbHlOYW1lPkxlZTwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5OYXJhbmcsIFNoYXJhbjwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+U2hhcmFuPC9naXZlbk5hbWU+CiAgICAgIDxmYW1pbHlOYW1lPk5hcmFuZzwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5NYXRlbmEsIE1pY2hhZWw8L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPk1pY2hhZWw8L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+TWF0ZW5hPC9mYW1pbHlOYW1lPgogICAgPC9jcmVhdG9yPgogICAgPGNyZWF0b3I+CiAgICAgIDxjcmVhdG9yTmFtZSBuYW1lVHlwZT0iUGVyc29uYWwiPlpob3UsIFlhbnFpPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5ZYW5xaTwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5aaG91PC9mYW1pbHlOYW1lPgogICAgPC9jcmVhdG9yPgogICAgPGNyZWF0b3I+CiAgICAgIDxjcmVhdG9yTmFtZSBuYW1lVHlwZT0iUGVyc29uYWwiPkxpLCBXZWk8L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPldlaTwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5MaTwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5MaXUsIFBldGVyIEouPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5QZXRlciBKLjwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5MaXU8L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgPC9jcmVhdG9ycz4KICA8dGl0bGVzPgogICAgPHRpdGxlPkV4cGxvcmluZyB0aGUgTGltaXRzIG9mIFRyYW5zZmVyIExlYXJuaW5nIHdpdGggYSBVbmlmaWVkIFRleHQtdG8tVGV4dCBUcmFuc2Zvcm1lcjwvdGl0bGU+CiAgPC90aXRsZXM+CiAgPHB1Ymxpc2hlcj5hclhpdjwvcHVibGlzaGVyPgogIDxwdWJsaWNhdGlvblllYXI+MjAxOTwvcHVibGljYXRpb25ZZWFyPgogIDxzdWJqZWN0cz4KICAgIDxzdWJqZWN0IHhtbDpsYW5nPSJlbiIgc3ViamVjdFNjaGVtZT0iYXJYaXYiPk1hY2hpbmUgTGVhcm5pbmcgKGNzLkxHKTwvc3ViamVjdD4KICAgIDxzdWJqZWN0IHhtbDpsYW5nPSJlbiIgc3ViamVjdFNjaGVtZT0iYXJYaXYiPkNvbXB1dGF0aW9uIGFuZCBMYW5ndWFnZSAoY3MuQ0wpPC9zdWJqZWN0PgogICAgPHN1YmplY3QgeG1sOmxhbmc9ImVuIiBzdWJqZWN0U2NoZW1lPSJhclhpdiI+TWFjaGluZSBMZWFybmluZyAoc3RhdC5NTCk8L3N1YmplY3Q+CiAgICA8c3ViamVjdCBzdWJqZWN0U2NoZW1lPSJGaWVsZHMgb2YgU2NpZW5jZSBhbmQgVGVjaG5vbG9neSAoRk9TKSI+Rk9TOiBDb21wdXRlciBhbmQgaW5mb3JtYXRpb24gc2NpZW5jZXM8L3N1YmplY3Q+CiAgPC9zdWJqZWN0cz4KICA8ZGF0ZXM+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iU3VibWl0dGVkIiBkYXRlSW5mb3JtYXRpb249InYxIj4yMDE5LTEwLTIzVDE3OjM3OjM2WjwvZGF0ZT4KICAgIDxkYXRlIGRhdGVUeXBlPSJVcGRhdGVkIiBkYXRlSW5mb3JtYXRpb249InYxIj4yMDE5LTEwLTI0VDAwOjE3OjE4WjwvZGF0ZT4KICAgIDxkYXRlIGRhdGVUeXBlPSJTdWJtaXR0ZWQiIGRhdGVJbmZvcm1hdGlvbj0idjIiPjIwMTktMTAtMjRUMTU6MTM6NTBaPC9kYXRlPgogICAgPGRhdGUgZGF0ZVR5cGU9IlVwZGF0ZWQiIGRhdGVJbmZvcm1hdGlvbj0idjIiPjIwMTktMTAtMjVUMDA6MTg6MDBaPC9kYXRlPgogICAgPGRhdGUgZGF0ZVR5cGU9IlN1Ym1pdHRlZCIgZGF0ZUluZm9ybWF0aW9uPSJ2MyI+MjAyMC0wNy0yOFQxMzoxMDowMVo8L2RhdGU+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iVXBkYXRlZCIgZGF0ZUluZm9ybWF0aW9uPSJ2MyI+MjAyMC0wNy0yOVQwMDoxNjo1Nlo8L2RhdGU+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iU3VibWl0dGVkIiBkYXRlSW5mb3JtYXRpb249InY0Ij4yMDIzLTA5LTE5VDE1OjE0OjQ4WjwvZGF0ZT4KICAgIDxkYXRlIGRhdGVUeXBlPSJVcGRhdGVkIiBkYXRlSW5mb3JtYXRpb249InY0Ij4yMDIzLTA5LTIwVDAwOjIwOjM0WjwvZGF0ZT4KICAgIDxkYXRlIGRhdGVUeXBlPSJBdmFpbGFibGUiIGRhdGVJbmZvcm1hdGlvbj0idjEiPjIwMTktMTA8L2RhdGU+CiAgPC9kYXRlcz4KICA8cmVzb3VyY2VUeXBlIHJlc291cmNlVHlwZUdlbmVyYWw9IlByZXByaW50Ij5BcnRpY2xlPC9yZXNvdXJjZVR5cGU+CiAgPHZlcnNpb24+NDwvdmVyc2lvbj4KICA8cmlnaHRzTGlzdD4KICAgIDxyaWdodHMgcmlnaHRzVVJJPSJodHRwOi8vYXJ4aXYub3JnL2xpY2Vuc2VzL25vbmV4Y2x1c2l2ZS1kaXN0cmliLzEuMC8iPmFyWGl2Lm9yZyBwZXJwZXR1YWwsIG5vbi1leGNsdXNpdmUgbGljZW5zZTwvcmlnaHRzPgogIDwvcmlnaHRzTGlzdD4KICA8ZGVzY3JpcHRpb25zPgogICAgPGRlc2NyaXB0aW9uIGRlc2NyaXB0aW9uVHlwZT0iQWJzdHJhY3QiPlRyYW5zZmVyIGxlYXJuaW5nLCB3aGVyZSBhIG1vZGVsIGlzIGZpcnN0IHByZS10cmFpbmVkIG9uIGEgZGF0YS1yaWNoIHRhc2sgYmVmb3JlIGJlaW5nIGZpbmUtdHVuZWQgb24gYSBkb3duc3RyZWFtIHRhc2ssIGhhcyBlbWVyZ2VkIGFzIGEgcG93ZXJmdWwgdGVjaG5pcXVlIGluIG5hdHVyYWwgbGFuZ3VhZ2UgcHJvY2Vzc2luZyAoTkxQKS4gVGhlIGVmZmVjdGl2ZW5lc3Mgb2YgdHJhbnNmZXIgbGVhcm5pbmcgaGFzIGdpdmVuIHJpc2UgdG8gYSBkaXZlcnNpdHkgb2YgYXBwcm9hY2hlcywgbWV0aG9kb2xvZ3ksIGFuZCBwcmFjdGljZS4gSW4gdGhpcyBwYXBlciwgd2UgZXhwbG9yZSB0aGUgbGFuZHNjYXBlIG9mIHRyYW5zZmVyIGxlYXJuaW5nIHRlY2huaXF1ZXMgZm9yIE5MUCBieSBpbnRyb2R1Y2luZyBhIHVuaWZpZWQgZnJhbWV3b3JrIHRoYXQgY29udmVydHMgYWxsIHRleHQtYmFzZWQgbGFuZ3VhZ2UgcHJvYmxlbXMgaW50byBhIHRleHQtdG8tdGV4dCBmb3JtYXQuIE91ciBzeXN0ZW1hdGljIHN0dWR5IGNvbXBhcmVzIHByZS10cmFpbmluZyBvYmplY3RpdmVzLCBhcmNoaXRlY3R1cmVzLCB1bmxhYmVsZWQgZGF0YSBzZXRzLCB0cmFuc2ZlciBhcHByb2FjaGVzLCBhbmQgb3RoZXIgZmFjdG9ycyBvbiBkb3plbnMgb2YgbGFuZ3VhZ2UgdW5kZXJzdGFuZGluZyB0YXNrcy4gQnkgY29tYmluaW5nIHRoZSBpbnNpZ2h0cyBmcm9tIG91ciBleHBsb3JhdGlvbiB3aXRoIHNjYWxlIGFuZCBvdXIgbmV3IGBgQ29sb3NzYWwgQ2xlYW4gQ3Jhd2xlZCBDb3JwdXMnJywgd2UgYWNoaWV2ZSBzdGF0ZS1vZi10aGUtYXJ0IHJlc3VsdHMgb24gbWFueSBiZW5jaG1hcmtzIGNvdmVyaW5nIHN1bW1hcml6YXRpb24sIHF1ZXN0aW9uIGFuc3dlcmluZywgdGV4dCBjbGFzc2lmaWNhdGlvbiwgYW5kIG1vcmUuIFRvIGZhY2lsaXRhdGUgZnV0dXJlIHdvcmsgb24gdHJhbnNmZXIgbGVhcm5pbmcgZm9yIE5MUCwgd2UgcmVsZWFzZSBvdXIgZGF0YSBzZXQsIHByZS10cmFpbmVkIG1vZGVscywgYW5kIGNvZGUuPC9kZXNjcmlwdGlvbj4KICA8L2Rlc2NyaXB0aW9ucz4KPC9yZXNvdXJjZT4=","url":"https://arxiv.org/abs/1910.10683","contentUrl":null,"metadataVersion":1,"schemaVersion":"http://datacite.org/schema/kernel-4","source":"mds","isActive":true,"state":"findable","reason":null,"viewCount":0,"viewsOverTime":[],"downloadCount":0,"downloadsOverTime":[],"referenceCount":0,"citationCount":4,"citationsOverTime":[{"year":"2022","total":3},{"year":"2024","total":1}],"partCount":0,"partOfCount":0,"versionCount":0,"versionOfCount":0,"created":"2022-02-27T11:33:27.000Z","registered":"2022-02-27T11:33:28.000Z","published":"2019","updated":"2024-03-14T08:42:47.000Z"},"relationships":{"client":{"data":{"id":"arxiv.content","type":"clients"}},"provider":{"data":{"id":"arxiv","type":"providers"}},"media":{"data":{"id":"10.48550/arxiv.1910.10683","type":"media"}},"references":{"data":[]},"citations":{"data":[{"id":"10.1145/3446132.3446403","type":"dois"},{"id":"10.1038/s42256-022-00457-9","type":"dois"},{"id":"10.1007/978-3-030-96623-2_14","type":"dois"},{"id":"10.4230/lipics.icdt.2024.1","type":"dois"}]},"parts":{"data":[]},"partOf":{"data":[]},"versions":{"data":[]},"versionOf":{"data":[]}}}}