[DLHacks]ニューラル固有表現抽出器を実装してみた話

H /2 2 1 3 L @himkt or @_makoh_ 3 2 D / @himkt /

6 A2 A C 6 :C6 C A6 A 2 6 ) C:CF 06 :C: 2 ( 6A6 6 C 6 AC 6A: 2 ( 2 C6A :2C: A( C2C: 2 : : C: ( : 2 6 2 6 6C 2 (2A 6 :6 .6 1 :E6A :CF G 2 6 06 62A : https://github.com/glample/tagger 2

• ec a ) 02 2 ( 1 0 2 10 E B • • N d E ( ( R 6 ;6 ; 3

• ) ) ( • • https://explosion.ai/demos/displacy-ent 4

• https://explosion.ai/demos/displacy-ent

• E •i • Br d ol Bi g s ) () George B-PER S I In O () Washington E-PER s : :E ( Visited O g E e ,) England S-LOC PERSON: George Washington, LOCATION: England 5

• 2 • 0 3 NE • • : Reuters Corpus train, dev: 10 : ECI Multilingual Text Corpus , test: 1 train, dev, test: 1 • 1. • http://trec.nist.gov/data/reuters/reuters.html 2. NE • https://www.clips.uantwerpen.be/conll2003/ner/ 3. NE • 2 https://www.clips.uantwerpen.be/conll2003/ner/ 6

• http://trec.nist.gov/data/reuters/reuters.html
• https://www.clips.uantwerpen.be/conll2003/ner/

•) ) 1S • NE • 0NE • 2NER • ) I prefixO L L 0 IC B prefix N L B prefixO :E 3 L ) • NE NE L B prefixO • ) ( • NE • NE B N B E N ELC 0 L 1 prefixO L S prefixO L IOB I-PER I-PER I-PER B-PER O I-LOC O BIO B-PER I-PER I-PER B-PER O I-LOC O BIOES B-PER I-PER E-PER S-PER O S-LOC O 7

• R 0 LC2 • 3 EB • E T 3 / N M S -) (LF 2 3 8

X = (x1 , x2 , x3 , x4 ),

:

Y = (y1 , y2 , y3 , y4 ),

:
<latexit sha1_base64="3DGdpXNuQh2Tm2FSOQvwYFyI1M8=">AAADa3ichVFNS9xQFD2Z2KpjW6e6EXUROsxgQeTFCopQEN0IdeFHR6c4MiTxjUbzRfJmaBzmD/gHuuhKwUVx7x9w486VC3+CiCuFbrroTSbSpuL0huTdc3LPue9ydc8yA8HYtZSRu1687O7pzfa9ev2mP/d2YD1w677BS4ZruX5Z1wJumQ4vCVNYvOz5XLN1i2/o+wvR/40G9wPTdT6L0ONbtrbjmDXT0ARR1dxZxdbErl5rlltK8aMy9gi/tqrquPIXmkyhDyk09X5cSUdRqVSyjxVf0tZhyjpMWYcp6/BZ62ouzyZYHMrTRE2SPJJYdnOXqGAbLgzUYYPDgaDcgoaAnk2oYPCI20ITe8Ro8AmZcQ1HC1nS14nnVKURu0/fHUKbCesQjnyD2MGgTha9PikVFNgV+8Hu2QU7ZTfs17Nezdgjuk9Ip97Wcq/afzi09vO/KptOgd0/qg4Kne7mEhbE6FTReT6BGmbiuUya04uZaGKj3atx8O1+bXa10CyyY3ZLsx6xa3ZO0zqNB+Nkha9+73gTO+pOC1X/Xd/TZH1yQqV8ZSo/N5+stgcjeIcx2t805rCIZZRgSKPSvPRJWsrcyYPysDzaLs1IiWYQqZALvwEsOdII</latexit>

h = Bi-LSTM(X),
Y = CRF(h),
where h = (h1 , h2 , h3 , h4 ).
<latexit sha1_base64="xp/pcaJ+VIRARYJ14oKeTdMeDig=">AAADxniclVHPS9xAFH4xrdVt1bVeCr2ELi4KdplYoaUgiELxUEF3XV0xsmTSWTM1v0hmd7Vhpef+Az2UHipoEf+MXrx56sE/ofRowYsHX7IRs0pXfCEz733zvm/m41HP4oEg5FTqkR887H3U1595/GRgcCg7/HQlcOu+wcqGa7l+heoBs7jDyoILi1U8n+k2tdgq3ZqLzlcbzA+46yyLHY9t2Pqmw2vc0AVC1WEpq9m6MGktNFtKflrRbOpuh7P85fvS8kJr7Oqw0hqfUO4TeUXTMlfstZT0XPHdtax5X9mUdCTWNJnPdlE9ZWJaSclX1QklVU12VK86qqnxQixdzeZIgcSh3E7UJMlBEotu9gQ0+AAuGFAHGxg4IDC3QIcAv3VQgYCH2AaE8BERHXyseNzDoAUZ5NcRZ9ilI7qF6yZW6wnqYB3pBrGCgTdZ+PvIVGCU/CaH5IwckyPyh1z8VyuMNaL37OBO21zmVYe+PCud38mycRdgXrO6MCi+zcVaIEKxo7s/ATV4E/vi6NOLkcix0b6r8enrWeltcTTMkz3yF73+IKfkF7p1Gv+M/SVW/Nb1JXZ0Ow5UvTm+28nKZEHFfGkqNzObjLYPnsMLGMP5vYYZmIdFKIMhNaXv0oH0U56XHbkuN9utPVLCGYGOkD9fAlv15MY=</latexit>

9

• • • Bi-LSTM 10

Ct = (c1 , c2 , . . . , cN ), ct = Bi-LSTM(Ct ), x t = wt <latexit sha1_base64="08WK82wpzxCGNSRvNdlIxrnue/0=">AAAD1XichVHPT9RAFH6lorj+YNGLiQcbN2wgATIlJhoTEsJePKABlmVJKGk6wyw70HaadnYBm94MBy8ePXDSxIPRq/4DXrx58sCfYDxiwsWDr92Ku7Ku07Tzvq/v+957eTRwRaQIOdaG9AvDFy+NXC5cuXrt+mhx7MZaJFsh4zUmXRmuUyfirvB5TQnl8vUg5I5HXV6nu5X0f73Nw0hIf1UdBHzTc7Z90RDMUUjZY9ody3NUkzbiSmIrozxnTPwmWGKbU0YXmkW0JVXUQz6ZnDLKhmUVurjMx/Ko3I8XxPRidfVxcuZasVWCkr6nx2f/zKeD91JsycBtRd31VR+vHp/m4H7SMpP928ms7GKJzJDsGOcDMw9KkJ8lWfwCFmyBBAYt8ICDDwpjFxyI8NkAEwgEyG1CDDvIOBAiElkOhwQKqG8hzzHLQXYXv9uINnLWR5z6RpkDw0ouviEqDRgnX8lbckI+k3fkG/n5T68480j7OcCbdrQ8sEef36qe/lfl4a2g+Uc1QEGxN4lYIUMxY/B8ChrwIJtL4JxBxqQTs06t9tOXJ9WHK+Nxmbwm33HWV+SYfMJp/fYP9maZrxwN7MRLq+NCzb/Xdz5Ym50xMV6+V5pfyFc7ArfhLkzg/u7DPDyCJagB0w6199oH7aNe1xP9mX7YSR3Scs1N6Dn6i1/YL/Fz</latexit> ct , ht = Bi-LSTM(xt ) 11

ct = Bi-LSTM(Ct ), <latexit sha1_base64="jeHdVRpNr8itLr1xK/IimYLSlZ4=">AAAC/HichVFNSxxBEH2OmhiNcWMugpchi4uCSm8IRARB9BAPBtR1VXBkmB57teN8MdO7aIb1B+QP5JBTQjwE8eofyCW3HMSDoD9APCrk4iE1sxNETdYaprvqdb2qehQPHBkpxk5atNa29kePO550dj3tftaTe967FPnV0BZl23f8cIVbkXCkJ8pKKkesBKGwXO6IZb41nbwv10QYSd9bVDuBWHOtDU9WpG0pgszcW8O11CavxHbdVHphQjdc7m/HU3JktrT4rj7493naVPWhYf2fVtANw8zl2ShLTb/vFDMnj8zm/NwvGFiHDxtVuBDwoMh3YCGibxVFMASErSHGe0IshBTJNEegjk7iVwkXlGURukXnBkWrGepRnNSN0go2dXLoD4mpY4Ads+/skv1k++ycXf+3VpzWSObZoZs3uCIwez72lX4/yHLpVti8YTVhcJrNp1gRwimjuT6FCsZSXZJ0BimSKLYbvWofPl2WxhcG4gL7yi5I6xd2wn6QWq92Ze/Ni4XPTSdxk+600OLd9d13ll6NFsmff52fnMpW24F+vMQg7e8NJjGDOZSp/gGOcIozbVf7pu1rB41UrSXjvMAt0w7/AH7MsO0=</latexit> - <latexit sha1_base64="s4ZyNEtDXXfCv7lae0UNP5ecY5A=">AAAECHiclVI9axRBGH43qzGumlwSC8Fm8chhFWYlEBWEYBoryYeXBLLHsTvO5dbsF7tza+Jy+QH5AxZWChZiYye22vgHLNLbiGUEGwufmVuJMeSCs+zM+z7v5zPv+GkY5JKxA2PEPHd+9MLYRevS5SvjE7XJqbU86WVcNHkSJtmG7+UiDGLRlIEMxUaaCS/yQ7Huby8q+3ohsjxI4kdyNxWtyNuKg07APQmoXdtzI092/U7Z7bel3bhnu4Xg5RGoUDdJw15uu1wUxy0N23WtMxM4pyZ42GjX6myW6WWfFJxKqFO1lpJJ4w659JgS4tSjiATFJCGH5FGOb5McYpQCa1FJT4B4lEELtI+gPlmI7wEX8PKAbmPfgrZZoTF0lTfXGTgqhfgzRNo0w76wN+yQfWZv2Tf269Rcpc6h+tnF6Q9iRdqe2L+2+vPMqAinpO5R1JAIH70l0CUQHx7D+Unq0G3NKwDPVCOKMR/UKp49P1y9uzJTNtgr9h1cX7ID9gls4+IHf70sVl4M7SQaUr3EnkFOwUvNYtCp8o5hfarvOdLMY0y3BM6BF/BSE21pJAPS0VPk6Fsx39G4YpCjqrL/uTuFF9rz/yPrqNj/67PwSp1/3+RJYe3WrAN5ea6+cL96r2N0nW7QTeSbpwV6QEvURAdfDcuYNq6a++Y78735YeA6YlQx03RsmR9/A9iE9a0=</latexit> ht = ~ht h~t ht = ~h1 h~N 12

Lample10 6 ( ) 0 • 0 2 0- ... TagLM [3]: 91.9, ELMo [4]: 92.2, BERT: 92.8... 13

14

15

• • • / E / : / - / - https://www.preferred-networks.jp/ja/pfn-logo https://github.com/RaRe-Technologies/gensim https://twitter.com/chakki_works 16

• https://www.preferred-networks.jp/ja/pfn-logo
• https://twitter.com/chakki_works
• https://github.com/RaRe-Technologies/gensim

Trainer Extension Updater Optimizer Iterator Dataset see also: https://qiita.com/mitmul/items/1e35fba085eb07a92560#trainer Model 17

Trainer Extension Updater Optimizer Model Iterator Dataset Transformer 18

• • • • • SequenceLabelingDataset, SequenceLabelingIterator: NamedEntityEvaluator: BIO(ES) NE -> seqeval LearningRateDecay: Ma+, 2016 DatasetTransformer: <-> id • tag2idx idx2tag • converter ( ) iconverter • • transformer: https://github.com/himkt/pyner/blob/master/pyner/named_entity/dataset.py#L23 • dataset: https://github.com/himkt/pyner/blob/master/pyner/named_entity/dataset.py#L58 • XavierInitializer: Lample+, 2016 • Chainer Glorot Uniform ... 19

• https://github.com/himkt/pyner/blob/master/pyner/named_entity/dataset.py#L23
• https://github.com/himkt/pyner/blob/master/pyner/named_entity/dataset.py#L58

• -03 p •S •t : •t : • ivL • •N s • o G p k [ ] e: 100 / 50 BiLSTM nd: 2x100 / 2x25 : 5.0 ratio: 0.5 ma k e 05 -2 , r C 20

• • • : 21

Setting Fscore Improvement Word only 69.76 – +char 82.62 +12.86 +pretrain 87.39 +17.63 +pretrain +char 87.76 +18.00 +pretrain +dropout 89.65 +19.89 90.81 +21.05 +pretrain +char +dropout <latexit sha1_base64="hv3dfV1YeZ+qdeHKq3b3kMe9Wgg=">AAAEuHichZG/b9NQEMevxkAxP5rCgsRiEbVCimo9u6qTdKpAQrC1DW0q1VWwnZf0UdvPen6OFKL8AyyMgJhAYkD8GSxsTAz9ExBjkVgYODsJKQ1Jz7J973yf7935vDhgiSTkeE65oF68dHn+inb12vUbC4XFm7sJT4VPd3wecLHnuQkNWER3JJMB3YsFdUMvoHXv6EH2vd6hImE8eiK7MT0I3XbEWsx3JYYai4rieLTNop50vTRwRb8XCNHXHMljkQZU06dbjUrJora+rOsPE58Lit7jMBa8Q0MaSd1xNCdkzXNk6lw0dR4F3UzHrhplO3NyW1nJNGZ0gFbyD12RERXLsK0RWjIto2LPpkv4n6RwWZTTZWO1OqbLhr06jR5z49rlU22XzIpByH/pU2hT8JinMqerhr02pqtGpXqG7jlea6LuSKKfoXlGlRgVsz9SGkCWaZC1fr4Lj0vJw3wdDo2afzeuNQpFgi1npk865tApwtA2eeErONAEDj6kEAKFCCT6AbiQ4LUPJhCIMXYAPXiGERcEnlieQ6EPGvIpxilmuRg9wmcbT/vDaITnTDfJFXysFOAtkNRhiXwjH8kJ+UI+ke/k91StXq6R9dPFtzdgadxYeHG79utcKsS3hMMxNYPwsDeOZ4kRDzNmzyehBZV8LoZzxnkkm9gf1Oo8f3VSW99e6i2T9+QHzvqOHJPPOG3U+el/2KLbb2d2EmbVcaHm2fVNOruWYaK/ZRU37g9XOw934C7cw/2VYQMewSbsgK8w5aXyWnmjrqtP1bbKBqnK3JC5Bf+YKv4A6kUTIQ==</latexit> 22

Setting Fscore Improvement Word only 69.76 – +char 82.62 +12.86 +pretrain 87.39 +17.63 +pretrain +char 87.76 +18.00 +pretrain +dropout 89.65 +19.89 90.81 +21.05 +pretrain +char +dropout <latexit sha1_base64="hv3dfV1YeZ+qdeHKq3b3kMe9Wgg=">AAAEuHichZG/b9NQEMevxkAxP5rCgsRiEbVCimo9u6qTdKpAQrC1DW0q1VWwnZf0UdvPen6OFKL8AyyMgJhAYkD8GSxsTAz9ExBjkVgYODsJKQ1Jz7J973yf7935vDhgiSTkeE65oF68dHn+inb12vUbC4XFm7sJT4VPd3wecLHnuQkNWER3JJMB3YsFdUMvoHXv6EH2vd6hImE8eiK7MT0I3XbEWsx3JYYai4rieLTNop50vTRwRb8XCNHXHMljkQZU06dbjUrJora+rOsPE58Lit7jMBa8Q0MaSd1xNCdkzXNk6lw0dR4F3UzHrhplO3NyW1nJNGZ0gFbyD12RERXLsK0RWjIto2LPpkv4n6RwWZTTZWO1OqbLhr06jR5z49rlU22XzIpByH/pU2hT8JinMqerhr02pqtGpXqG7jlea6LuSKKfoXlGlRgVsz9SGkCWaZC1fr4Lj0vJw3wdDo2afzeuNQpFgi1npk865tApwtA2eeErONAEDj6kEAKFCCT6AbiQ4LUPJhCIMXYAPXiGERcEnlieQ6EPGvIpxilmuRg9wmcbT/vDaITnTDfJFXysFOAtkNRhiXwjH8kJ+UI+ke/k91StXq6R9dPFtzdgadxYeHG79utcKsS3hMMxNYPwsDeOZ4kRDzNmzyehBZV8LoZzxnkkm9gf1Oo8f3VSW99e6i2T9+QHzvqOHJPPOG3U+el/2KLbb2d2EmbVcaHm2fVNOruWYaK/ZRU37g9XOw934C7cw/2VYQMewSbsgK8w5aXyWnmjrqtP1bbKBqnK3JC5Bf+YKv4A6kUTIQ==</latexit> 23

Setting Fscore Improvement Word only 69.76 – +char 82.62 +12.86 +pretrain 87.39 +17.63 +pretrain +char 87.76 +18.00 +pretrain +dropout 89.65 +19.89 90.81 +21.05 +pretrain +char +dropout <latexit sha1_base64="hv3dfV1YeZ+qdeHKq3b3kMe9Wgg=">AAAEuHichZG/b9NQEMevxkAxP5rCgsRiEbVCimo9u6qTdKpAQrC1DW0q1VWwnZf0UdvPen6OFKL8AyyMgJhAYkD8GSxsTAz9ExBjkVgYODsJKQ1Jz7J973yf7935vDhgiSTkeE65oF68dHn+inb12vUbC4XFm7sJT4VPd3wecLHnuQkNWER3JJMB3YsFdUMvoHXv6EH2vd6hImE8eiK7MT0I3XbEWsx3JYYai4rieLTNop50vTRwRb8XCNHXHMljkQZU06dbjUrJora+rOsPE58Lit7jMBa8Q0MaSd1xNCdkzXNk6lw0dR4F3UzHrhplO3NyW1nJNGZ0gFbyD12RERXLsK0RWjIto2LPpkv4n6RwWZTTZWO1OqbLhr06jR5z49rlU22XzIpByH/pU2hT8JinMqerhr02pqtGpXqG7jlea6LuSKKfoXlGlRgVsz9SGkCWaZC1fr4Lj0vJw3wdDo2afzeuNQpFgi1npk865tApwtA2eeErONAEDj6kEAKFCCT6AbiQ4LUPJhCIMXYAPXiGERcEnlieQ6EPGvIpxilmuRg9wmcbT/vDaITnTDfJFXysFOAtkNRhiXwjH8kJ+UI+ke/k91StXq6R9dPFtzdgadxYeHG79utcKsS3hMMxNYPwsDeOZ4kRDzNmzyehBZV8LoZzxnkkm9gf1Oo8f3VSW99e6i2T9+QHzvqOHJPPOG3U+el/2KLbb2d2EmbVcaHm2fVNOruWYaK/ZRU37g9XOw934C7cw/2VYQMewSbsgK8w5aXyWnmjrqtP1bbKBqnK3JC5Bf+YKv4A6kUTIQ==</latexit> 24

Setting Fscore Improvement Word only 69.76 – +char 82.62 +12.86 +pretrain 87.39 +17.63 +pretrain +char 87.76 +18.00 +pretrain +dropout 89.65 +19.89 90.81 +21.05 +pretrain +char +dropout <latexit sha1_base64="hv3dfV1YeZ+qdeHKq3b3kMe9Wgg=">AAAEuHichZG/b9NQEMevxkAxP5rCgsRiEbVCimo9u6qTdKpAQrC1DW0q1VWwnZf0UdvPen6OFKL8AyyMgJhAYkD8GSxsTAz9ExBjkVgYODsJKQ1Jz7J973yf7935vDhgiSTkeE65oF68dHn+inb12vUbC4XFm7sJT4VPd3wecLHnuQkNWER3JJMB3YsFdUMvoHXv6EH2vd6hImE8eiK7MT0I3XbEWsx3JYYai4rieLTNop50vTRwRb8XCNHXHMljkQZU06dbjUrJora+rOsPE58Lit7jMBa8Q0MaSd1xNCdkzXNk6lw0dR4F3UzHrhplO3NyW1nJNGZ0gFbyD12RERXLsK0RWjIto2LPpkv4n6RwWZTTZWO1OqbLhr06jR5z49rlU22XzIpByH/pU2hT8JinMqerhr02pqtGpXqG7jlea6LuSKKfoXlGlRgVsz9SGkCWaZC1fr4Lj0vJw3wdDo2afzeuNQpFgi1npk865tApwtA2eeErONAEDj6kEAKFCCT6AbiQ4LUPJhCIMXYAPXiGERcEnlieQ6EPGvIpxilmuRg9wmcbT/vDaITnTDfJFXysFOAtkNRhiXwjH8kJ+UI+ke/k91StXq6R9dPFtzdgadxYeHG79utcKsS3hMMxNYPwsDeOZ4kRDzNmzyehBZV8LoZzxnkkm9gf1Oo8f3VSW99e6i2T9+QHzvqOHJPPOG3U+el/2KLbb2d2EmbVcaHm2fVNOruWYaK/ZRU37g9XOw934C7cw/2VYQMewSbsgK8w5aXyWnmjrqtP1bbKBqnK3JC5Bf+YKv4A6kUTIQ==</latexit> 25

Setting Fscore Improvement Word only 69.76 – +char 82.62 +12.86 +pretrain 87.39 +17.63 +pretrain +char 87.76 +18.00 +pretrain +dropout 89.65 +19.89 90.81 +21.05 +pretrain +char +dropout <latexit sha1_base64="hv3dfV1YeZ+qdeHKq3b3kMe9Wgg=">AAAEuHichZG/b9NQEMevxkAxP5rCgsRiEbVCimo9u6qTdKpAQrC1DW0q1VWwnZf0UdvPen6OFKL8AyyMgJhAYkD8GSxsTAz9ExBjkVgYODsJKQ1Jz7J973yf7935vDhgiSTkeE65oF68dHn+inb12vUbC4XFm7sJT4VPd3wecLHnuQkNWER3JJMB3YsFdUMvoHXv6EH2vd6hImE8eiK7MT0I3XbEWsx3JYYai4rieLTNop50vTRwRb8XCNHXHMljkQZU06dbjUrJora+rOsPE58Lit7jMBa8Q0MaSd1xNCdkzXNk6lw0dR4F3UzHrhplO3NyW1nJNGZ0gFbyD12RERXLsK0RWjIto2LPpkv4n6RwWZTTZWO1OqbLhr06jR5z49rlU22XzIpByH/pU2hT8JinMqerhr02pqtGpXqG7jlea6LuSKKfoXlGlRgVsz9SGkCWaZC1fr4Lj0vJw3wdDo2afzeuNQpFgi1npk865tApwtA2eeErONAEDj6kEAKFCCT6AbiQ4LUPJhCIMXYAPXiGERcEnlieQ6EPGvIpxilmuRg9wmcbT/vDaITnTDfJFXysFOAtkNRhiXwjH8kJ+UI+ke/k91StXq6R9dPFtzdgadxYeHG79utcKsS3hMMxNYPwsDeOZ4kRDzNmzyehBZV8LoZzxnkkm9gf1Oo8f3VSW99e6i2T9+QHzvqOHJPPOG3U+el/2KLbb2d2EmbVcaHm2fVNOruWYaK/ZRU37g9XOw934C7cw/2VYQMewSbsgK8w5aXyWnmjrqtP1bbKBqnK3JC5Bf+YKv4A6kUTIQ==</latexit> 26

Setting Fscore Improvement Word only 69.76 – +char 82.62 +12.86 +pretrain 87.39 +17.63 +pretrain +char 87.76 +18.00 +pretrain +dropout 89.65 +19.89 90.81 +21.05 +pretrain +char +dropout <latexit sha1_base64="hv3dfV1YeZ+qdeHKq3b3kMe9Wgg=">AAAEuHichZG/b9NQEMevxkAxP5rCgsRiEbVCimo9u6qTdKpAQrC1DW0q1VWwnZf0UdvPen6OFKL8AyyMgJhAYkD8GSxsTAz9ExBjkVgYODsJKQ1Jz7J973yf7935vDhgiSTkeE65oF68dHn+inb12vUbC4XFm7sJT4VPd3wecLHnuQkNWER3JJMB3YsFdUMvoHXv6EH2vd6hImE8eiK7MT0I3XbEWsx3JYYai4rieLTNop50vTRwRb8XCNHXHMljkQZU06dbjUrJora+rOsPE58Lit7jMBa8Q0MaSd1xNCdkzXNk6lw0dR4F3UzHrhplO3NyW1nJNGZ0gFbyD12RERXLsK0RWjIto2LPpkv4n6RwWZTTZWO1OqbLhr06jR5z49rlU22XzIpByH/pU2hT8JinMqerhr02pqtGpXqG7jlea6LuSKKfoXlGlRgVsz9SGkCWaZC1fr4Lj0vJw3wdDo2afzeuNQpFgi1npk865tApwtA2eeErONAEDj6kEAKFCCT6AbiQ4LUPJhCIMXYAPXiGERcEnlieQ6EPGvIpxilmuRg9wmcbT/vDaITnTDfJFXysFOAtkNRhiXwjH8kJ+UI+ke/k91StXq6R9dPFtzdgadxYeHG79utcKsS3hMMxNYPwsDeOZ4kRDzNmzyehBZV8LoZzxnkkm9gf1Oo8f3VSW99e6i2T9+QHzvqOHJPPOG3U+el/2KLbb2d2EmbVcaHm2fVNOruWYaK/ZRU37g9XOw934C7cw/2VYQMewSbsgK8w5aXyWnmjrqtP1bbKBqnK3JC5Bf+YKv4A6kUTIQ==</latexit> 27

• ) ) ) )( • s fE N • BiLSTM-CRF ho • cR Nm t • di g ) E er f E • Chainer di • u R N • Eho a ATrainer E P l n 28

himkt/pyner https://speakerdeck.com/himkt/implement-neural-named-entity-tagger 29

• https://speakerdeck.com/himkt/implement-neural-named-entity-tagger

• • CoNLL2003 • • 0 : 246,679 -> 210,023 https://github.com/glample/tagger/blob/master/utils.py#L82 30

• https://github.com/glample/tagger/blob/master/utils.py#L82

Setting Fscore Improvement +pretrain +char +dropout 90.81 – +pretrain +char +dropout +BIO 89.61 -1.20 +pretrain +char +dropout -clipping 90.42 -0.39 +pretrain +char +dropout +lower 90.50 -0.31 +pretrain +char +dropout -zero 90.78 -0.03 <latexit sha1_base64="NOJ3MWEfD5yZyARr/Bw8Vb9vn3Y=">AAAEb3ichZHNbtNAEIAnboBifprCASQktCJqBYoSrVOgCaeqSBU9NW1IW6muItvZpEttr7VeB7VRXoAX4MAJJA6IJ+DMhRsnDn0ExAkViQsHxo4JpZXDWPbOjOebnx07cHmoKD3KaVP5c+cvTF/UL12+cnWmMHttMxSRdFjLEa6Q27YVMpf7rKW4ctl2IJnl2S7bsvcfx/+3+kyGXPhP1UHAdj2r5/MudyyFrvZs7oNpsx73B8qyI9eSw4Er5VA3lQhk5DKdZEuTKcX9HpknZCV0hGSorXqBFH3mMV8R09RNj3dOpxmYdpeUsEklLe6TkrNnSVLqSBGISA3jbElEnVZqRmLGUi7H6UZ6FktKy6trMVCrVx4aY9KoVGkCZ3Jlx+VBkI6Cde9XxzCtLNT/VM6u64rnTKbwA3oSNsZtZ1c/ZFKk8GLtBEwXkiu0hVLCS27RZH5nvCi9XShiUCLkrGKkShFSaYjCZzChAwIciMADBj4o1F2wIMRnBwygEKBvFwbwDD0WSLR4EsNgCDryEfoZRlno3cdvD62d1OujHecNkwwOVnLxlUgSmKNf6Dt6TD/R9/Qr/ZWZa5DkiPs5wNMesSxoz7y42fz5X8rDU8HeX2oCYWNvAm2FHhsjJs+noAu1ZC6OcwaJJ57YGdXqH748bj7amBvM0zf0G876mh7Rjzit3//hvF1nG68mduLF1XGhxun1nVU2qxUD9fVqcWk5Xe003II7cBf3twhL8AQa0AJHI9qKtqY1pr7nb+Rv58koVMulzHX4R/L3fgPquBTF</latexit> 31

32

https://github.com/glample/tagger/blob/master/utils.py#L44 33

• https://github.com/glample/tagger/blob/master/utils.py#L44

Xavier initialization [7] https://github.com/glample/tagger/blob/master/utils.py#L44 34

• https://github.com/glample/tagger/blob/master/utils.py#L44

https://github.com/glample/tagger/blob/master/model.py#L281 https://github.com/chainer/chainer/blob/master/chainer/links/loss/crf1d.py https://github.com/chainer/chainer/pull/5807 35

• https://github.com/glample/tagger/blob/master/model.py#L281
• https://github.com/chainer/chainer/blob/master/chainer/links/loss/crf1d.py

• 7 • [-sqrt(6/(in+out)), sqrt(6/(in+out))] (Xavier initialization [7]) • Lample+ [1]: https://github.com/glample/tagger/blob/master/utils.py#L52 • [-sqrt(3/dim), sqrt(3/dim)] (LeCun initialization [8]) • Ma+ [2]: https://github.com/XuezheMax/NeuroNLP/blob/master/sequence_labeling.py#L161 • Liu+ [6]: https://github.com/LiyuanLucasLiu/LM-LSTM-CRF/blob/master/model/utils.py#L793 • ( :Glorot ) • word2vec/word2vec: [-(0.5/dim), (0.5/dim)] • https://github.com/tmikolov/word2vec/blob/master/word2vec.c#L365 • facebookresearch/fastText: [-(1/dim), (1/dim)] • https://github.com/facebookresearch/fastText/blob/master/src/fasttext.cc#L734 • 7 : C 36

• https://github.com/tmikolov/word2vec/blob/master/word2vec.c#L365
• https://github.com/facebookresearch/fastText/blob/master/src/fasttext.cc#L734

Setting Improvement Word only 78.05 +8.29 +char 84.04 +14.28 +pretrain 86.93 +17.17 +pretrain +char 88.51 +18.75 +pretrain +dropout 89.91 +20.15 90.99 +21.23 +pretrain +char +dropout +BIO 89.99 +20.23 +pretrain +char +dropout -clipping 90.60 +20.84 +pretrain +char +dropout +lower 90.52 +20.76 +pretrain +char +dropout -zero 90.25 +20.49 +pretrain +char +dropout <latexit sha1_base64="LPxSfcjquR40IRP7qiUJSpuLdoA=">AAAFrnichZK/b9NAFMdfGwIl/GgLCxKLRdUKKeJ0dpPGZqqKhOhEf9AfUl1FtnNNTc8+63wJaqOMLAwwMjCBxID4M1jYmBj6JyDGIsHAwLPj1IGS9Fm23z2/z/e95zs34n6sKD0eGy9cKF68NHG5dOXqteuTU9M3NmPRkh7b8AQXctt1Ysb9kG0oX3G2HUnmBC5nW+7Bg+T7VpvJ2BfhE3UYsd3AaYb+nu85CkP16fFftsuafthRjtvijux2uOTdkq1EJFuclbQRts6U8sOmNqdpD2NPSIbechBJ0WYBC5Vm2yU78Bvn6GwJ2dBEyA8TnZpJaDVxUiubxLASmVFdYJq378gEMiuEVvp0Wa8QwxxNl/FfKen4YUovEGs+p2tErw2jcy6vbZKqntMmqVX/Sw+gDSki0VIpbRErpw1K9Jzu2O7emYp9uJtAaYZFiWV1+xo9yNCJMd/N+ximopWXlh/3+7AG+zDmU3ooeM/jfhRlpwBbWKCDtFnp1x5emYtnTGZ01RikawunnQ+vf8SkyGijOkhX0pNju0IpEaRn0GZh4/Scl+pTM5TQ1LSzjp45M5DZipj6AjY0QIAHLQiAQQgKfQ4OxHjtgA4UIoztQgeeYsQBiSs/zWHQhRLyLYwzzHIweoDPJq52smiI60Q3ThU8rMTxlkhqMEu/0g/0hH6mH+k3+nuoVifVSPo5xLfbY1lUn3xxa/3nuVSAbwX7OTWCcLE3gWuFERczRs+nYA/MdC4f54zSSDKx16vVPnp9sn5/bbYzR9/R7zjrW3pMP+G0YfuH936Vrb0Z2UmQVMcN1f/dvrPOpkF09FeNmcWlbGsn4Dbcgbu4fzVYhEewAhvgFeqF54WXhVdFWtws7hbrvdTxsYy5CX9Zcf8PZKxUlQ==</latexit> Fscore 37

5APM G M DE A PMAN B M 5 A I E T 7A CIE E I 3 GA A G 5 -3 I AI 8ALPAI A 3 AGEIC E E EMA E I G 38 4 -55N -70 4 A G -3 8A E NP AM ENA NALPAI A CCEIC RE D E EMA E I G G ICP CA AGN 6A AMN A G -3 .AA I AS P GEUA R M MA MANAI E IN 6A AMN A G 5 -3 RAM 8ALPAI A 3 AGEIC RE D NF R MA 5APM G 3 ICP CA 4 AG 3EP A G 2 5 GG I AS N MA MA A ALP G A AM R M MA MANAI E IN RE D ME GA 3EIC A G 4536 :I AMN I EIC DA EBBE PG T B M EIEIC AA BAA B MR M IAPM G IA R MFN 1G M A G 28 8 BBE EAI F M 3A-PI A G 5APM G 5A R MF ME FN B DA M A AI E I 38