{"data":{"id":"10.48550/arxiv.1905.06922","type":"dois","attributes":{"doi":"10.48550/arxiv.1905.06922","prefix":"10.48550","suffix":"arxiv.1905.06922","identifiers":[{"identifier":"1905.06922","identifierType":"arXiv"}],"alternateIdentifiers":[{"alternateIdentifierType":"arXiv","alternateIdentifier":"1905.06922"}],"creators":[{"name":"Poole, Ben","nameType":"Personal","givenName":"Ben","familyName":"Poole","affiliation":[],"nameIdentifiers":[]},{"name":"Ozair, Sherjil","nameType":"Personal","givenName":"Sherjil","familyName":"Ozair","affiliation":[],"nameIdentifiers":[]},{"name":"Oord, Aaron van den","nameType":"Personal","givenName":"Aaron van den","familyName":"Oord","affiliation":[],"nameIdentifiers":[]},{"name":"Alemi, Alexander A.","nameType":"Personal","givenName":"Alexander A.","familyName":"Alemi","affiliation":[],"nameIdentifiers":[]},{"name":"Tucker, George","nameType":"Personal","givenName":"George","familyName":"Tucker","affiliation":[],"nameIdentifiers":[]}],"titles":[{"title":"On Variational Bounds of Mutual Information"}],"publisher":"arXiv","container":{},"publicationYear":2019,"subjects":[{"lang":"en","subject":"Machine Learning (cs.LG)","subjectScheme":"arXiv"},{"lang":"en","subject":"Machine Learning (stat.ML)","subjectScheme":"arXiv"},{"subject":"FOS: Computer and information sciences","subjectScheme":"Fields of Science and Technology (FOS)"},{"subject":"FOS: Computer and information sciences","schemeUri":"http://www.oecd.org/science/inno/38235147.pdf","subjectScheme":"Fields of Science and Technology (FOS)"}],"contributors":[],"dates":[{"date":"2019-05-16T17:31:53Z","dateType":"Submitted","dateInformation":"v1"},{"date":"2019-05-17T00:21:59Z","dateType":"Updated","dateInformation":"v1"},{"date":"2019-05","dateType":"Available","dateInformation":"v1"},{"date":"2019","dateType":"Issued"}],"language":null,"types":{"ris":"GEN","bibtex":"misc","citeproc":"article","schemaOrg":"CreativeWork","resourceType":"Article","resourceTypeGeneral":"Preprint"},"relatedIdentifiers":[],"relatedItems":[],"sizes":[],"formats":[],"version":"1","rightsList":[{"rights":"arXiv.org perpetual, non-exclusive license","rightsUri":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/"}],"descriptions":[{"description":"Estimating and optimizing Mutual Information (MI) is core to many problems in machine learning; however, bounding MI in high dimensions is challenging. To establish tractable and scalable objectives, recent work has turned to variational bounds parameterized by neural networks, but the relationships and tradeoffs between these bounds remains unclear. In this work, we unify these recent developments in a single framework. We find that the existing variational lower bounds degrade when the MI is large, exhibiting either high bias or high variance. To address this problem, we introduce a continuum of lower bounds that encompasses previous bounds and flexibly trades off bias and variance. On high-dimensional, controlled problems, we empirically characterize the bias and variance of the bounds and their gradients and demonstrate the effectiveness of our new bounds for estimation and representation learning.","descriptionType":"Abstract"},{"description":"ICML 2019","descriptionType":"Other"}],"geoLocations":[],"fundingReferences":[],"xml":"PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPHJlc291cmNlIHhtbG5zPSJodHRwOi8vZGF0YWNpdGUub3JnL3NjaGVtYS9rZXJuZWwtNCIgeG1sbnM6eHNpPSJodHRwOi8vd3d3LnczLm9yZy8yMDAxL1hNTFNjaGVtYS1pbnN0YW5jZSIgeHNpOnNjaGVtYUxvY2F0aW9uPSJodHRwOi8vZGF0YWNpdGUub3JnL3NjaGVtYS9rZXJuZWwtNCBodHRwOi8vc2NoZW1hLmRhdGFjaXRlLm9yZy9tZXRhL2tlcm5lbC00LjMvbWV0YWRhdGEueHNkIj4KICA8aWRlbnRpZmllciBpZGVudGlmaWVyVHlwZT0iRE9JIj4xMC40ODU1MC9BUlhJVi4xOTA1LjA2OTIyPC9pZGVudGlmaWVyPgogIDxhbHRlcm5hdGVJZGVudGlmaWVycz4KICAgIDxhbHRlcm5hdGVJZGVudGlmaWVyIGFsdGVybmF0ZUlkZW50aWZpZXJUeXBlPSJhclhpdiI+MTkwNS4wNjkyMjwvYWx0ZXJuYXRlSWRlbnRpZmllcj4KICA8L2FsdGVybmF0ZUlkZW50aWZpZXJzPgogIDxjcmVhdG9ycz4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5Qb29sZSwgQmVuPC9jcmVhdG9yTmFtZT4KICAgICAgPGdpdmVuTmFtZT5CZW48L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+UG9vbGU8L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+T3phaXIsIFNoZXJqaWw8L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPlNoZXJqaWw8L2dpdmVuTmFtZT4KICAgICAgPGZhbWlseU5hbWU+T3phaXI8L2ZhbWlseU5hbWU+CiAgICA8L2NyZWF0b3I+CiAgICA8Y3JlYXRvcj4KICAgICAgPGNyZWF0b3JOYW1lIG5hbWVUeXBlPSJQZXJzb25hbCI+T29yZCwgQWFyb24gdmFuIGRlbjwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+QWFyb24gdmFuIGRlbjwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5Pb3JkPC9mYW1pbHlOYW1lPgogICAgPC9jcmVhdG9yPgogICAgPGNyZWF0b3I+CiAgICAgIDxjcmVhdG9yTmFtZSBuYW1lVHlwZT0iUGVyc29uYWwiPkFsZW1pLCBBbGV4YW5kZXIgQS48L2NyZWF0b3JOYW1lPgogICAgICA8Z2l2ZW5OYW1lPkFsZXhhbmRlciBBLjwvZ2l2ZW5OYW1lPgogICAgICA8ZmFtaWx5TmFtZT5BbGVtaTwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICAgIDxjcmVhdG9yPgogICAgICA8Y3JlYXRvck5hbWUgbmFtZVR5cGU9IlBlcnNvbmFsIj5UdWNrZXIsIEdlb3JnZTwvY3JlYXRvck5hbWU+CiAgICAgIDxnaXZlbk5hbWU+R2VvcmdlPC9naXZlbk5hbWU+CiAgICAgIDxmYW1pbHlOYW1lPlR1Y2tlcjwvZmFtaWx5TmFtZT4KICAgIDwvY3JlYXRvcj4KICA8L2NyZWF0b3JzPgogIDx0aXRsZXM+CiAgICA8dGl0bGU+T24gVmFyaWF0aW9uYWwgQm91bmRzIG9mIE11dHVhbCBJbmZvcm1hdGlvbjwvdGl0bGU+CiAgPC90aXRsZXM+CiAgPHB1Ymxpc2hlcj5hclhpdjwvcHVibGlzaGVyPgogIDxwdWJsaWNhdGlvblllYXI+MjAxOTwvcHVibGljYXRpb25ZZWFyPgogIDxzdWJqZWN0cz4KICAgIDxzdWJqZWN0IHhtbDpsYW5nPSJlbiIgc3ViamVjdFNjaGVtZT0iYXJYaXYiPk1hY2hpbmUgTGVhcm5pbmcgKGNzLkxHKTwvc3ViamVjdD4KICAgIDxzdWJqZWN0IHhtbDpsYW5nPSJlbiIgc3ViamVjdFNjaGVtZT0iYXJYaXYiPk1hY2hpbmUgTGVhcm5pbmcgKHN0YXQuTUwpPC9zdWJqZWN0PgogICAgPHN1YmplY3Qgc3ViamVjdFNjaGVtZT0iRmllbGRzIG9mIFNjaWVuY2UgYW5kIFRlY2hub2xvZ3kgKEZPUykiPkZPUzogQ29tcHV0ZXIgYW5kIGluZm9ybWF0aW9uIHNjaWVuY2VzPC9zdWJqZWN0PgogIDwvc3ViamVjdHM+CiAgPGRhdGVzPgogICAgPGRhdGUgZGF0ZVR5cGU9IlN1Ym1pdHRlZCIgZGF0ZUluZm9ybWF0aW9uPSJ2MSI+MjAxOS0wNS0xNlQxNzozMTo1M1o8L2RhdGU+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iVXBkYXRlZCIgZGF0ZUluZm9ybWF0aW9uPSJ2MSI+MjAxOS0wNS0xN1QwMDoyMTo1OVo8L2RhdGU+CiAgICA8ZGF0ZSBkYXRlVHlwZT0iQXZhaWxhYmxlIiBkYXRlSW5mb3JtYXRpb249InYxIj4yMDE5LTA1PC9kYXRlPgogIDwvZGF0ZXM+CiAgPHJlc291cmNlVHlwZSByZXNvdXJjZVR5cGVHZW5lcmFsPSJQcmVwcmludCI+QXJ0aWNsZTwvcmVzb3VyY2VUeXBlPgogIDx2ZXJzaW9uPjE8L3ZlcnNpb24+CiAgPHJpZ2h0c0xpc3Q+CiAgICA8cmlnaHRzIHJpZ2h0c1VSST0iaHR0cDovL2FyeGl2Lm9yZy9saWNlbnNlcy9ub25leGNsdXNpdmUtZGlzdHJpYi8xLjAvIj5hclhpdi5vcmcgcGVycGV0dWFsLCBub24tZXhjbHVzaXZlIGxpY2Vuc2U8L3JpZ2h0cz4KICA8L3JpZ2h0c0xpc3Q+CiAgPGRlc2NyaXB0aW9ucz4KICAgIDxkZXNjcmlwdGlvbiBkZXNjcmlwdGlvblR5cGU9IkFic3RyYWN0Ij5Fc3RpbWF0aW5nIGFuZCBvcHRpbWl6aW5nIE11dHVhbCBJbmZvcm1hdGlvbiAoTUkpIGlzIGNvcmUgdG8gbWFueSBwcm9ibGVtcyBpbiBtYWNoaW5lIGxlYXJuaW5nOyBob3dldmVyLCBib3VuZGluZyBNSSBpbiBoaWdoIGRpbWVuc2lvbnMgaXMgY2hhbGxlbmdpbmcuIFRvIGVzdGFibGlzaCB0cmFjdGFibGUgYW5kIHNjYWxhYmxlIG9iamVjdGl2ZXMsIHJlY2VudCB3b3JrIGhhcyB0dXJuZWQgdG8gdmFyaWF0aW9uYWwgYm91bmRzIHBhcmFtZXRlcml6ZWQgYnkgbmV1cmFsIG5ldHdvcmtzLCBidXQgdGhlIHJlbGF0aW9uc2hpcHMgYW5kIHRyYWRlb2ZmcyBiZXR3ZWVuIHRoZXNlIGJvdW5kcyByZW1haW5zIHVuY2xlYXIuIEluIHRoaXMgd29yaywgd2UgdW5pZnkgdGhlc2UgcmVjZW50IGRldmVsb3BtZW50cyBpbiBhIHNpbmdsZSBmcmFtZXdvcmsuIFdlIGZpbmQgdGhhdCB0aGUgZXhpc3RpbmcgdmFyaWF0aW9uYWwgbG93ZXIgYm91bmRzIGRlZ3JhZGUgd2hlbiB0aGUgTUkgaXMgbGFyZ2UsIGV4aGliaXRpbmcgZWl0aGVyIGhpZ2ggYmlhcyBvciBoaWdoIHZhcmlhbmNlLiBUbyBhZGRyZXNzIHRoaXMgcHJvYmxlbSwgd2UgaW50cm9kdWNlIGEgY29udGludXVtIG9mIGxvd2VyIGJvdW5kcyB0aGF0IGVuY29tcGFzc2VzIHByZXZpb3VzIGJvdW5kcyBhbmQgZmxleGlibHkgdHJhZGVzIG9mZiBiaWFzIGFuZCB2YXJpYW5jZS4gT24gaGlnaC1kaW1lbnNpb25hbCwgY29udHJvbGxlZCBwcm9ibGVtcywgd2UgZW1waXJpY2FsbHkgY2hhcmFjdGVyaXplIHRoZSBiaWFzIGFuZCB2YXJpYW5jZSBvZiB0aGUgYm91bmRzIGFuZCB0aGVpciBncmFkaWVudHMgYW5kIGRlbW9uc3RyYXRlIHRoZSBlZmZlY3RpdmVuZXNzIG9mIG91ciBuZXcgYm91bmRzIGZvciBlc3RpbWF0aW9uIGFuZCByZXByZXNlbnRhdGlvbiBsZWFybmluZy48L2Rlc2NyaXB0aW9uPgogICAgPGRlc2NyaXB0aW9uIGRlc2NyaXB0aW9uVHlwZT0iT3RoZXIiPklDTUwgMjAxOTwvZGVzY3JpcHRpb24+CiAgPC9kZXNjcmlwdGlvbnM+CjwvcmVzb3VyY2U+","url":"https://arxiv.org/abs/1905.06922","contentUrl":null,"metadataVersion":0,"schemaVersion":"http://datacite.org/schema/kernel-4","source":"mds","isActive":true,"state":"findable","reason":null,"viewCount":0,"viewsOverTime":[],"downloadCount":0,"downloadsOverTime":[],"referenceCount":0,"citationCount":0,"citationsOverTime":[],"partCount":0,"partOfCount":0,"versionCount":0,"versionOfCount":0,"created":"2022-02-27T22:41:10.000Z","registered":"2022-02-27T22:41:11.000Z","published":"2019","updated":"2022-02-27T22:41:11.000Z"},"relationships":{"client":{"data":{"id":"arxiv.content","type":"clients"}},"provider":{"data":{"id":"arxiv","type":"providers"}},"media":{"data":{"id":"10.48550/arxiv.1905.06922","type":"media"}},"references":{"data":[]},"citations":{"data":[]},"parts":{"data":[]},"partOf":{"data":[]},"versions":{"data":[]},"versionOf":{"data":[]}}}}