Retrieval Augmented Language Model Pre-Training
 |
 |
abstract = {Language model pre-training has been shown to capture a surprising amount of world knowledge, crucial for NLP tasks such as question answering. However, this knowledge is stored implicitly in the parameters of a neural network, requiring ever-larger networks to cover more facts. To capture knowledge in a more modular and interpretable way, we augment language model pre-training with a latent knowledge retriever, which allows the model to retrieve and attend over documents from a large corpus such as Wikipedia, used during pre-training, fine-tuning and inference. For the first time, we show how to pre-train such a knowledge retriever in an unsupervised manner, using masked language modeling as the learning signal and backpropagating through a retrieval step that considers millions of documents. We demonstrate the effectiveness of Retrieval-Augmented Language Model pre-training (REALM) by fine-tuning on the challenging task of Open-domain Question Answering (Open-QA). We compare against state-of-the-art models for both explicit and implicit knowledge storage on three popular Open-QA benchmarks, and find that we outperform all previous methods by a significant margin (4-16% absolute accuracy), while also providing qualitative benefits such as interpretability and modularity.},
apice = {RealmIcml2020},
author = {Kelvin Guu and Kenton Lee and Zora Tung and Panupong Pasupat and Mingwei Chang},
booktitle = {37th International Conference on Machine Learning (ICML 2020)},
editor = {Hal Daumé III and Aarti Singh},
month = {13-18 July},
numpages = 10,
pages = {3929--3938},
publisher = {PMLR},
series = {Proceedings of Machine Learning Research},
title = {Retrieval Augmented Language Model Pre-Training},
url = {https://proceedings.mlr.press/v119/guu20a.html},
urlpdf = {https://proceedings.mlr.press/v119/guu20a/guu20a.pdf},
volume = 119,
year = 2020
}