<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-01742442</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-19T11:13:41+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">The Role of Information Complexity and Randomization in Representation Learning</title>
            <author role="aut">
              <persName>
                <forename type="first">Matias</forename>
                <surname>Vera</surname>
              </persName>
              <idno type="halauthorid">1332672-0</idno>
              <affiliation ref="#struct-92878"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Pablo</forename>
                <surname>Piantanida</surname>
              </persName>
              <email type="md5">4d99eb094c1c3424e1f0788dcb8f365d</email>
              <email type="domain">centralesupelec.fr</email>
              <idno type="idhal" notation="string">pablo-piantanida</idno>
              <idno type="idhal" notation="numeric">736967</idno>
              <idno type="halauthorid" notation="string">28498-736967</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-8717-2117</idno>
              <idno type="IDREF">https://www.idref.fr/116306300</idno>
              <affiliation ref="#struct-1289"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Leonardo Rey</forename>
                <surname>Vega</surname>
              </persName>
              <idno type="halauthorid">1332673-0</idno>
              <affiliation ref="#struct-92878"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Pablo</forename>
                <surname>Piantanida</surname>
              </persName>
              <email type="md5">4d99eb094c1c3424e1f0788dcb8f365d</email>
              <email type="domain">centralesupelec.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2018-03-24 23:42:59</date>
              <date type="whenModified">2024-03-18 03:05:32</date>
              <date type="whenReleased">2018-03-24 23:42:59</date>
              <date type="whenProduced">2018-03-24</date>
              <ref type="externalLink" target="http://arxiv.org/pdf/1802.05355"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="329762">
                <persName>
                  <forename>Pablo</forename>
                  <surname>Piantanida</surname>
                </persName>
                <email type="md5">4d99eb094c1c3424e1f0788dcb8f365d</email>
                <email type="domain">centralesupelec.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-01742442</idno>
            <idno type="halUri">https://centralesupelec.hal.science/hal-01742442</idno>
            <idno type="halBibtex">vera:hal-01742442</idno>
            <idno type="halRefHtml">2018</idno>
            <idno type="halRef">2018</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="UNIV-PSUD">Université Paris Sud - Paris XI</idno>
            <idno type="stamp" n="SUP_LSS">Laboratoire des Signaux et Systèmes</idno>
            <idno type="stamp" n="SUP_TELECOMS" corresp="SUP_LSS">Pôle Télécoms et Réseaux</idno>
            <idno type="stamp" n="CENTRALESUPELEC">Ecole CentraleSupélec</idno>
            <idno type="stamp" n="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="UNIV-PSUD-SACLAY" corresp="UNIV-PARIS-SACLAY">Université Paris Sud pour Paris Saclay</idno>
            <idno type="stamp" n="CENTRALESUPELEC-SACLAY" corresp="UNIV-PARIS-SACLAY">CENTRALESUPELEC-SACLAY</idno>
            <idno type="stamp" n="GS-ENGINEERING">Graduate School Sciences de l'Ingénierie et des Systèmes</idno>
            <idno type="stamp" n="GS-COMPUTER-SCIENCE">Graduate School Computer Science</idno>
          </seriesStmt>
          <notesStmt>
            <note type="commentary">35 pages, 3 figures. Submitted for publication</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">The Role of Information Complexity and Randomization in Representation Learning</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Matias</forename>
                    <surname>Vera</surname>
                  </persName>
                  <idno type="halauthorid">1332672-0</idno>
                  <affiliation ref="#struct-92878"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Pablo</forename>
                    <surname>Piantanida</surname>
                  </persName>
                  <email type="md5">4d99eb094c1c3424e1f0788dcb8f365d</email>
                  <email type="domain">centralesupelec.fr</email>
                  <idno type="idhal" notation="string">pablo-piantanida</idno>
                  <idno type="idhal" notation="numeric">736967</idno>
                  <idno type="halauthorid" notation="string">28498-736967</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-8717-2117</idno>
                  <idno type="IDREF">https://www.idref.fr/116306300</idno>
                  <affiliation ref="#struct-1289"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Leonardo Rey</forename>
                    <surname>Vega</surname>
                  </persName>
                  <idno type="halauthorid">1332673-0</idno>
                  <affiliation ref="#struct-92878"/>
                </author>
              </analytic>
              <monogr>
                <imprint/>
              </monogr>
              <idno type="arxiv">1802.05355</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="info.info-it">Computer Science [cs]/Information Theory [cs.IT]</classCode>
              <classCode scheme="halDomain" n="info.info-ni">Computer Science [cs]/Networking and Internet Architecture [cs.NI]</classCode>
              <classCode scheme="halDomain" n="math.math-it">Mathematics [math]/Information Theory [math.IT]</classCode>
              <classCode scheme="halDomain" n="math.math-st">Mathematics [math]/Statistics [math.ST]</classCode>
              <classCode scheme="halTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halOldTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halTreeTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>A grand challenge in representation learning is to learn the different explanatory factors of variation behind the high dimen- sional data. Encoder models are often determined to optimize performance on training data when the real objective is to generalize well to unseen data. Although there is enough numerical evidence suggesting that noise injection (during training) at the representation level might improve the generalization ability of encoders, an information-theoretic understanding of this principle remains elusive. This paper presents a sample-dependent bound on the generalization gap of the cross-entropy loss that scales with the information complexity (IC) of the representations, meaning the mutual information between inputs and their representations. The IC is empirically investigated for standard multi-layer neural networks with SGD on MNIST and CIFAR-10 datasets; the behaviour of the gap and the IC appear to be in direct correlation, suggesting that SGD selects encoders to implicitly minimize the IC. We specialize the IC to study the role of Dropout on the generalization capacity of deep encoders which is shown to be directly related to the encoder capacity, being a measure of the distinguishability among samples from their representations. Our results support some recent regularization methods.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="institution" xml:id="struct-92878" status="VALID">
          <idno type="ROR">https://ror.org/03cqe8w59</idno>
          <orgName>Consejo Nacional de Investigaciones Científicas y Técnicas [Buenos Aires]</orgName>
          <orgName type="acronym">CONICET</orgName>
          <desc>
            <address>
              <addrLine>Avda. Rivadavia 1917 - CP C1033AAJ - Cdad. de Buenos Aires</addrLine>
              <country key="AR"/>
            </address>
            <ref type="url">http://www.conicet.gov.ar/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-1289" status="OLD">
          <idno type="IdRef">133818799</idno>
          <idno type="RNSR">199812842X</idno>
          <orgName>Laboratoire des signaux et systèmes</orgName>
          <orgName type="acronym">L2S</orgName>
          <date type="end">2019-12-31</date>
          <desc>
            <address>
              <addrLine>Plateau de Moulon 3 rue Joliot Curie 91192 GIF SUR YVETTE CEDEX</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.l2s.centralesupelec.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-92966" type="direct"/>
            <relation active="#struct-411575" type="direct"/>
            <relation name="UMR8506" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-92966" status="OLD">
          <idno type="ROR">https://ror.org/028rypz17</idno>
          <orgName>Université Paris-Sud - Paris 11</orgName>
          <orgName type="acronym">UP11</orgName>
          <date type="end">2019-12-31</date>
          <desc>
            <address>
              <addrLine>Bâtiment 300 - 91405 Orsay cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.u-psud.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-411575" status="VALID">
          <idno type="IdRef">184443237</idno>
          <idno type="ROR">https://ror.org/019tcpt25</idno>
          <orgName>CentraleSupélec</orgName>
          <desc>
            <address>
              <addrLine>3, rue Joliot Curie,Plateau de Moulon,91192 GIF-SUR-YVETTE Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.centralesupelec.fr</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>