@inproceedings{hong-etal-2023-visual,
title = {Visual Coherence Loss for Coherent and Visually Grounded Story Generation},
author = {Xudong Hong and Vera Demberg and Asad Sayeed and Qiankun Zheng and Bernt Schiele},
editor = {Anna Rogers and Jordan Boyd-Graber and Naoaki Okazaki},
url = {https://aclanthology.org/2023.findings-acl.603},
doi = {https://doi.org/10.18653/v1/2023.findings-acl.603},
year = {2023},
date = {2023},
booktitle = {Findings of the Association for Computational Linguistics: ACL 2023},
pages = {9456-9470},
publisher = {Association for Computational Linguistics},
address = {Toronto, Canada},
abstract = {Local coherence is essential for long-form text generation models. We identify two important aspects of local coherence within the visual storytelling task: (1) the model needs to represent re-occurrences of characters within the image sequence in order to mention them correctly in the story; (2) character representations should enable us to find instances of the same characters and distinguish different characters. In this paper, we propose a loss function inspired by a linguistic theory of coherence for self-supervised learning for image sequence representations. We further propose combining features from an object and a face detector to construct stronger character features. To evaluate input-output relevance that current reference-based metrics don{'}t measure, we propose a character matching metric to check whether the models generate referring expressions correctly for characters in input image sequences. Experiments on a visual story generation dataset show that our proposed features and loss function are effective for generating more coherent and visually grounded stories.},
pubstate = {published},
type = {inproceedings}
}