Publications

During referential communication, speaker choices regarding the syntactic encoding of their expressions can modulate the linear ordering of the properties necessary to identify the referent. We investigated whether such syntactic choices are influenced by the informativity of these properties in a given visual context, as quantified by Referential Entropy Reduction (RER). In two experiments, a maze-based sentence completion task was used to examine whether informativity of a particular property (animal or action) influenced the decision to produce pre- versus post-nominal modifications when describing animal-performing-action referents in a visual scene. While many participants used a fixed strategy, informativity did significantly influence linearization for the remaining participants, consistent with a maximal informativity strategy in which the high RER property is be encoded first. This suggests that speakers who vary their encodings are indeed sensitive to the informativity of properties in a visual scene, preferring syntactic linearization in which informative properties appear early.

• https://escholarship.org/uc/item/95v6j0sx

@inproceedings{Muqing-etal-2023,
title = {Does informativity modulate linearization preferences in reference production? },
author = {Muqing Li and Noortje Venhuizen and Torsten Jachmann and Heiner Drenhaus and Matthew W. Crocker},
url = {https://escholarship.org/uc/item/95v6j0sx},
year = {2023},
date = {2023},
booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society},
pages = {3028-3054},
abstract = {During referential communication, speaker choices regarding the syntactic encoding of their expressions can modulate the linear ordering of the properties necessary to identify the referent. We investigated whether such syntactic choices are influenced by the informativity of these properties in a given visual context, as quantified by Referential Entropy Reduction (RER). In two experiments, a maze-based sentence completion task was used to examine whether informativity of a particular property (animal or action) influenced the decision to produce pre- versus post-nominal modifications when describing animal-performing-action referents in a visual scene. While many participants used a fixed strategy, informativity did significantly influence linearization for the remaining participants, consistent with a maximal informativity strategy in which the high RER property is be encoded first. This suggests that speakers who vary their encodings are indeed sensitive to the informativity of properties in a visual scene, preferring syntactic linearization in which informative properties appear early.},
pubstate = {published},
type = {inproceedings}
}

Copy BibTeX to Clipboard

Behavioral studies have shown that speaker gaze to objects in a co-present scene can influence listeners’ expectations about how the utterance will unfold. These findings have recently been supported by ERP studies that linked the underlying mechanisms of the integration of speaker gaze with an utterance meaning representation to multiple ERP components. This leads to the question, however, as to whether speaker gaze should be considered part of the communicative signal itself, such that the referential information conveyed by gaze can help listeners not only form expectations but also to confirm referential expectations induced by the prior linguistic context. In the current study, we investigated this question by conducting an ERP experiment (N=24, Age:[19,31]), in which referential expectations were established by linguistic context together with several depicted objects in the scene. Those expectations then could be confirmed by subsequent speaker gaze that preceded the referential expression. Participants were presented with a centrally positioned face performing gaze actions aligned to utterances comparing two out of three displayed objects, with the task to judge whether the sentence was true given the provided scene. We manipulated the gaze cue to be either Present (toward the subsequently named object) or Absent preceding contextually Expected or Unexpected referring nouns. The results provided strong evidence for gaze as being treated as an integral part of the communicative signal: While in the absence of gaze, effects of phonological verification (PMN), word meaning retrieval (N400) and sentence meaning integration/evaluation (P600) were found on the unexpected noun, in the presence of gaze effects of retrieval (N400) and integration/evaluation (P300) were solely found in response to the pre-referent gaze cue when it was directed toward the unexpected referent with attenuated effects on the following referring noun.

• https://www.sciencedirect.com/science/article/pii/S0010027723000835?via%3Dihub
• https://doi.org/10.1016/j.cognition.2023.105449

@article{Jachmannetal-23,
title = {When a look is enough: Neurophysiological correlates of referential speaker gaze in situated comprehension},
author = {Torsten Jachmann and Heiner Drenhaus and Maria Staudte and Matthew W. Crocker},
url = {https://www.sciencedirect.com/science/article/pii/S0010027723000835?via%3Dihub},
doi = {https://doi.org/10.1016/j.cognition.2023.105449},
year = {2023},
date = {2023},
journal = {Cognition},
pages = {105449},
volume = {236},
abstract = {Behavioral studies have shown that speaker gaze to objects in a co-present scene can influence listeners’ expectations about how the utterance will unfold. These findings have recently been supported by ERP studies that linked the underlying mechanisms of the integration of speaker gaze with an utterance meaning representation to multiple ERP components. This leads to the question, however, as to whether speaker gaze should be considered part of the communicative signal itself, such that the referential information conveyed by gaze can help listeners not only form expectations but also to confirm referential expectations induced by the prior linguistic context. In the current study, we investigated this question by conducting an ERP experiment (N=24, Age:[19,31]), in which referential expectations were established by linguistic context together with several depicted objects in the scene. Those expectations then could be confirmed by subsequent speaker gaze that preceded the referential expression. Participants were presented with a centrally positioned face performing gaze actions aligned to utterances comparing two out of three displayed objects, with the task to judge whether the sentence was true given the provided scene. We manipulated the gaze cue to be either Present (toward the subsequently named object) or Absent preceding contextually Expected or Unexpected referring nouns. The results provided strong evidence for gaze as being treated as an integral part of the communicative signal: While in the absence of gaze, effects of phonological verification (PMN), word meaning retrieval (N400) and sentence meaning integration/evaluation (P600) were found on the unexpected noun, in the presence of gaze effects of retrieval (N400) and integration/evaluation (P300) were solely found in response to the pre-referent gaze cue when it was directed toward the unexpected referent with attenuated effects on the following referring noun.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

Decades of studies trying to define the extent to which artificial neural networks can exhibit systematicity suggest that systematicity can be achieved by connectionist models but not by default. Here we present a novel connectionist model of sentence production that employs rich situation model representations originally proposed for modeling systematicity in comprehension. The high performance of our model demonstrates that such representations are also well suited to model language production. Furthermore, the model can produce multiple novel sentences for previously unseen situations, including in a different voice (actives vs. passive) and with words in new syntactic roles, thus demonstrating semantic and syntactic generalization and arguably systematicity. Our results provide yet further evidence that such connectionist approaches can achieve systematicity, in production as well as comprehension. We propose our positive results to be a consequence of the regularities of the microworld from which the semantic representations are derived, which provides a sufficient structure from which the neural network can interpret novel inputs.

• https://doi.org/10.3390/info12080329
• https://doi.org/10.3390/info12080329

@article{Calvillo2021semantic,
title = {Semantic Systematicity in Connectionist Language Production},
author = {Jesús Calvillo and Harm Brouwer and Matthew W. Crocker},
url = {https://doi.org/10.3390/info12080329},
doi = {https://doi.org/10.3390/info12080329},
year = {2021},
date = {2021},
journal = {Information (Special issue on Neural Natural Language Generation)},
pages = {329},
volume = {12},
number = {8},
abstract = {Decades of studies trying to define the extent to which artificial neural networks can exhibit systematicity suggest that systematicity can be achieved by connectionist models but not by default. Here we present a novel connectionist model of sentence production that employs rich situation model representations originally proposed for modeling systematicity in comprehension. The high performance of our model demonstrates that such representations are also well suited to model language production. Furthermore, the model can produce multiple novel sentences for previously unseen situations, including in a different voice (actives vs. passive) and with words in new syntactic roles, thus demonstrating semantic and syntactic generalization and arguably systematicity. Our results provide yet further evidence that such connectionist approaches can achieve systematicity, in production as well as comprehension. We propose our positive results to be a consequence of the regularities of the microworld from which the semantic representations are derived, which provides a sufficient structure from which the neural network can interpret novel inputs.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

In referential communication, Grice’s Maxim of Quantity is thought to imply that utterances conveying unnecessary information should incur comprehension difficulties. There is, however, considerable evidence that speakers frequently encode redundant information in their referring expressions, raising the question as to whether such overspecifications hinder listeners’ processing. Evidence from previous work is inconclusive, and mostly comes from offline studies. In this article, we present two event-related potential (ERP) experiments, investigating the real-time comprehension of referring expressions that contain redundant adjectives in complex visual contexts. Our findings provide support for both Gricean and bounded-rational accounts. We argue that these seemingly incompatible results can be reconciled if common ground is taken into account. We propose a bounded-rational account of overspecification, according to which even redundant words can be beneficial to comprehension to the extent that they facilitate the reduction of listeners’ uncertainty regarding the target referent.

• https://doi.org/10.1111/cogs.13071
• https://doi.org/10.1111/cogs.13071

@article{Tourtouri2021rational,
title = {Rational Redundancy in Referring Expressions: Evidence from Event-related Potentials},
author = {Elli Tourtouri and Francesca Delogu and Matthew W. Crocker},
url = {https://doi.org/10.1111/cogs.13071},
doi = {https://doi.org/10.1111/cogs.13071},
year = {2021},
date = {2021-12-12},
journal = {Cognitive Science},
pages = {e13071},
publisher = {Wiley},
volume = {45},
number = {12},
abstract = {In referential communication, Grice's Maxim of Quantity is thought to imply that utterances conveying unnecessary information should incur comprehension difficulties. There is, however, considerable evidence that speakers frequently encode redundant information in their referring expressions, raising the question as to whether such overspecifications hinder listeners’ processing. Evidence from previous work is inconclusive, and mostly comes from offline studies. In this article, we present two event-related potential (ERP) experiments, investigating the real-time comprehension of referring expressions that contain redundant adjectives in complex visual contexts. Our findings provide support for both Gricean and bounded-rational accounts. We argue that these seemingly incompatible results can be reconciled if common ground is taken into account. We propose a bounded-rational account of overspecification, according to which even redundant words can be beneficial to comprehension to the extent that they facilitate the reduction of listeners’ uncertainty regarding the target referent.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

The current work investigates how preexisting mental representations of the meaning of an utterance (top-down processing) affect the comprehension of external perceptual properties of the linguistic input (bottom-up processing). When it comes to top-down bottom-up processing in the brain previous findings report a division of focus between left and right hemispheric mechanisms. The PARLO sentence comprehension model posits that the LH employs top-down mechanisms which allow for efficient anticipatory processing, while the RH relies more on bottom-up mechanisms. A shortcoming of the PARLO model is that it’s based on experiments manipulating solely top-down contextual constraint, leading to conclusions that hemispheric asymmetries are a function of differences in the efficiency of top-down rather than bottom-up mechanisms. Up until now, there has been no investigation of asymmetries in bottom-up processing, nor an investigation of the potential interactions between that and top-down processing for each hemisphere. This thesis consists of four event-related potential (ERP) experiments divided into two parts. Experiments 1 (central presentation) and 2 (hemispheric presentation) manipulate the bottom-up lexical frequency of critical words in high and low predictability contexts. Experiments 3 (central presentation) and 4 (hemispheric presentation) manipulate bottom-up word status, presenting critical words and pseudowords in the same high and low predictability contexts. The results allow us to extend previous findings and present the Spotlight Theory of Hemispheric Comprehension. We argue that the LH employs a kind of spotlight focus, which affords very efficient top-down processing of the expected input, since only highly predictable inputs receive additional facilitation based their bottom-up features. Alternatively, the RH lack of spotlight mechanism and focus on bottom-up lexical properties allows for the reliable processing of less predictable and irregular inputs. In combination, these complementary processing strategies provide the comprehension system with the efficiency and robustness required in a wide range of communicative situations.

• https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/31806
• https://doi.org/https://dx.doi.org/10.22028/D291-33976

@phdthesis{Vergilova_Diss_2021,
title = {The Lateralization of Expectations: Hemispheric Differences in Top-down and Bottom-up Word Processing in Context},
author = {Yoana Vergilova},
url = {https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/31806},
doi = {https://doi.org/https://dx.doi.org/10.22028/D291-33976},
year = {2021},
date = {2021},
school = {Saarland University},
address = {Saarbruecken, Germany},
abstract = {The current work investigates how preexisting mental representations of the meaning of an utterance (top-down processing) affect the comprehension of external perceptual properties of the linguistic input (bottom-up processing). When it comes to top-down bottom-up processing in the brain previous findings report a division of focus between left and right hemispheric mechanisms. The PARLO sentence comprehension model posits that the LH employs top-down mechanisms which allow for efficient anticipatory processing, while the RH relies more on bottom-up mechanisms. A shortcoming of the PARLO model is that it’s based on experiments manipulating solely top-down contextual constraint, leading to conclusions that hemispheric asymmetries are a function of differences in the efficiency of top-down rather than bottom-up mechanisms. Up until now, there has been no investigation of asymmetries in bottom-up processing, nor an investigation of the potential interactions between that and top-down processing for each hemisphere. This thesis consists of four event-related potential (ERP) experiments divided into two parts. Experiments 1 (central presentation) and 2 (hemispheric presentation) manipulate the bottom-up lexical frequency of critical words in high and low predictability contexts. Experiments 3 (central presentation) and 4 (hemispheric presentation) manipulate bottom-up word status, presenting critical words and pseudowords in the same high and low predictability contexts. The results allow us to extend previous findings and present the Spotlight Theory of Hemispheric Comprehension. We argue that the LH employs a kind of spotlight focus, which affords very efficient top-down processing of the expected input, since only highly predictable inputs receive additional facilitation based their bottom-up features. Alternatively, the RH lack of spotlight mechanism and focus on bottom-up lexical properties allows for the reliable processing of less predictable and irregular inputs. In combination, these complementary processing strategies provide the comprehension system with the efficiency and robustness required in a wide range of communicative situations.},
pubstate = {published},
type = {phdthesis}
}

Copy BibTeX to Clipboard

Rational Speech Act theory (Frank & Goodman, 2012) has been successfully applied in numerous communicative settings, including studies using one-shot web-based language games. Several follow-up studies of the latter, however, suggest that listeners may not behave as pragmatically as originally suggested in those tasks. We investigate whether, in such reference games, listeners’ pragmatic reasoning about an informative speaker is improved by greater exposure to the task, and/or prior experience with being a speaker in this task. While we find limited evidence that increased exposure results in more pragmatic responses, listeners do show increased pragmatic reasoning after playing the role of the speaker. Moreover, we find that only in the Speaker-first condition, participant’s tendency to be an informative speaker predicts their degree of pragmatic behavior as a listener. These findings demonstrate that, in these settings, experience as a speaker enhances the ability of listeners to reason pragmatically, as modeled by RSA.

• https://escholarship.org/uc/item/0xc7f7wc

@inproceedings{sikos2021speak,
title = {Speak before you listen: Pragmatic reasoning in multi-trial language games},
author = {Les Sikos and Noortje Venhuizen and Heiner Drenhaus and Matthew W. Crocker},
url = {https://escholarship.org/uc/item/0xc7f7wc},
year = {2021},
date = {2021},
booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society},
abstract = {Rational Speech Act theory (Frank & Goodman, 2012) has been successfully applied in numerous communicative settings, including studies using one-shot web-based language games. Several follow-up studies of the latter, however, suggest that listeners may not behave as pragmatically as originally suggested in those tasks. We investigate whether, in such reference games, listeners’ pragmatic reasoning about an informative speaker is improved by greater exposure to the task, and/or prior experience with being a speaker in this task. While we find limited evidence that increased exposure results in more pragmatic responses, listeners do show increased pragmatic reasoning after playing the role of the speaker. Moreover, we find that only in the Speaker-first condition, participant’s tendency to be an informative speaker predicts their degree of pragmatic behavior as a listener. These findings demonstrate that, in these settings, experience as a speaker enhances the ability of listeners to reason pragmatically, as modeled by RSA.},
pubstate = {published},
type = {inproceedings}
}

Copy BibTeX to Clipboard

Natural language semantics has recently sought to combine the complementary strengths of formal and distributional approaches to meaning. However, given the fundamentally different ‘representational currency’ underlying these approaches—models of the world versus linguistic co-occurrence—their unification has proven extremely difficult. Here, we define Distributional Formal Semantics, which integrates distributionality into a formal semantic system on the level of formal models. This approach offers probabilistic, distributed meaning representations that are inherently compositional, and that naturally capture fundamental semantic notions such as quantification and entailment. Furthermore, we show how the probabilistic nature of these representations allows for probabilistic inference, and how the information-theoretic notion of “information” (measured in Entropy and Surprisal) naturally follows from it. Finally, we illustrate how meaning representations can be derived incrementally from linguistic input using a recurrent neural network model, and how the resultant incremental semantic construction procedure intuitively captures key semantic phenomena, including negation, presupposition, and anaphoricity.

• https://www.sciencedirect.com/science/article/pii/S089054012100078X
• https://doi.org/10.1016/j.ic.2021.104763

@article{venhuizen2021distributional,
title = {Distributional formal semantics},
author = {Noortje Venhuizen and Petra Hendriks and Matthew W. Crocker and Harm Brouwer},
url = {https://www.sciencedirect.com/science/article/pii/S089054012100078X},
doi = {https://doi.org/10.1016/j.ic.2021.104763},
year = {2021},
date = {2021},
journal = {Information and Computation},
pages = {104763},
abstract = {Natural language semantics has recently sought to combine the complementary strengths of formal and distributional approaches to meaning. However, given the fundamentally different ‘representational currency’ underlying these approaches—models of the world versus linguistic co-occurrence—their unification has proven extremely difficult. Here, we define Distributional Formal Semantics, which integrates distributionality into a formal semantic system on the level of formal models. This approach offers probabilistic, distributed meaning representations that are inherently compositional, and that naturally capture fundamental semantic notions such as quantification and entailment. Furthermore, we show how the probabilistic nature of these representations allows for probabilistic inference, and how the information-theoretic notion of “information” (measured in Entropy and Surprisal) naturally follows from it. Finally, we illustrate how meaning representations can be derived incrementally from linguistic input using a recurrent neural network model, and how the resultant incremental semantic construction procedure intuitively captures key semantic phenomena, including negation, presupposition, and anaphoricity.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

This paper reports on two experiments that partially replicate an experiment by Fraundorf and Watson (2011, J Mem. Lang.) on the recall effect of filler particles. Their subjects listened to three passages of a story, either with or without filler particles, which they had to retell afterwards. They analysed the subject‘ retelling in terms of whether important plot points were remem- bered or not. For their English data, they found that filler parti- cles facilitate the recall of the plot points significantly compared to stories that did not include filler particles. As this seems to be a convincing experimental design, we aimed at evaluating this method as a web-based experiment which may, if found to be suitable, easily be applied to other languages. Furthermore, we investigated whether their results are found in German as well (Experiment 1), and evaluated whether filler duration has an ef- fect on recall performance (Experiment 2). Our results could not replicate the findings of the original study: in fact, the op- posite effect was found for German. In Experiment 1, participants performed better on recall in the fluent condition, while no significant results were found for English in Experiment 2.

• Muhlack_et_al_2021     (0.19MB)
• https://www.isca-speech.org/archive/interspeech_2021/muhlack21_interspeech.html
• https://doi.org/10.21437/Interspeech.2021-1056

@inproceedings{Muhlack2021,
title = {Revisiting recall effects of filler particles in German and English},
author = {Beeke Muhlack and Mikey Elmers and Heiner Drenhaus and Marjolein van Os and Raphael Werner and Margarita Ryzhova and Bernd M{\"o}bius},
url = {https://www.isca-speech.org/archive/interspeech_2021/muhlack21_interspeech.html},
doi = {https://doi.org/10.21437/Interspeech.2021-1056},
year = {2021},
date = {2021},
booktitle = {Proceedings of Interspeech 2021},
pages = {3979-3983},
publisher = {Interspeech},
address = {Brno, Czechia},
abstract = {This paper reports on two experiments that partially replicate an experiment by Fraundorf and Watson (2011, J Mem. Lang.) on the recall effect of filler particles. Their subjects listened to three passages of a story, either with or without filler particles, which they had to retell afterwards. They analysed the subject' retelling in terms of whether important plot points were remem- bered or not. For their English data, they found that filler parti- cles facilitate the recall of the plot points significantly compared to stories that did not include filler particles. As this seems to be a convincing experimental design, we aimed at evaluating this method as a web-based experiment which may, if found to be suitable, easily be applied to other languages. Furthermore, we investigated whether their results are found in German as well (Experiment 1), and evaluated whether filler duration has an ef- fect on recall performance (Experiment 2). Our results could not replicate the findings of the original study: in fact, the op- posite effect was found for German. In Experiment 1, participants performed better on recall in the fluent condition, while no significant results were found for English in Experiment 2.},
pubstate = {published},
type = {inproceedings}
}

Copy BibTeX to Clipboard

Recent work has shown that linguistic and visual contexts jointly modulate linguistic expectancy and, thus, the processing effort for a (more or less) expected critical word (Ankener et al., 2018; Tourtouri et al., 2019; Staudte et al., 2020). According to these findings, uncertainty about the upcoming referent in a visually-situated sentence can be reduced by exploiting the selectional restrictions of a preceding word (e.g., a verb or an adjective), which then reduces processing effort on the critical word (e.g., a referential noun). Interestingly, however, no such modulation was observed in these studies on the expectation-generating word itself. The goal of the current study is to investigate whether the reduction of uncertainty (i.e., the generation of expectations) simply does not modulate processing effort — or whether the particular subject-verb-object (SVO) sentence structure used in these studies (which emphasizes the referential nature of the noun as direct pointer to visually co-present objects) accounts for the observed pattern. To test these questions, the current design reverses the functional roles of nouns and verbs by using sentence constructions in which the noun reduces uncertainty about upcoming verbs, and the verb provides the disambiguating and reference-resolving piece of information. Experiment~1 (a Visual World Paradigm study) and Experiment~2 (a Grammaticality Maze study) both replicate the effect found in Ankener et al. (2018) of visually-situated context on the word which uniquely identifies the referent, albeit on the verb in the current study. Results on the noun, where uncertainty is reduced and expectations are generated in the current design, were mixed and were most likely influenced by design decisions specific to each experiment. These results show that processing of the reference-resolving word — whether it be a noun or a verb — reliably benefits from the prior linguistic and visual information that lead to the generation of concrete expectations.

• https://www.frontiersin.org/articles/10.3389/fpsyg.2021.661898/full
• https://doi.org/10.3389/fpsyg.2021.661898

@article{Sikos2021b,
title = {A rose by any other verb: The effect of expectations and word category on processing effort in situated sentence comprehension},
author = {Les Sikos and Maria Staudte},
url = {https://www.frontiersin.org/articles/10.3389/fpsyg.2021.661898/full},
doi = {https://doi.org/10.3389/fpsyg.2021.661898},
year = {2021},
date = {2021},
journal = {Frontiers in Psychology},
abstract = {Recent work has shown that linguistic and visual contexts jointly modulate linguistic expectancy and, thus, the processing effort for a (more or less) expected critical word (Ankener et al., 2018; Tourtouri et al., 2019; Staudte et al., 2020). According to these findings, uncertainty about the upcoming referent in a visually-situated sentence can be reduced by exploiting the selectional restrictions of a preceding word (e.g., a verb or an adjective), which then reduces processing effort on the critical word (e.g., a referential noun). Interestingly, however, no such modulation was observed in these studies on the expectation-generating word itself. The goal of the current study is to investigate whether the reduction of uncertainty (i.e., the generation of expectations) simply does not modulate processing effort --- or whether the particular subject-verb-object (SVO) sentence structure used in these studies (which emphasizes the referential nature of the noun as direct pointer to visually co-present objects) accounts for the observed pattern. To test these questions, the current design reverses the functional roles of nouns and verbs by using sentence constructions in which the noun reduces uncertainty about upcoming verbs, and the verb provides the disambiguating and reference-resolving piece of information. Experiment~1 (a Visual World Paradigm study) and Experiment~2 (a Grammaticality Maze study) both replicate the effect found in Ankener et al. (2018) of visually-situated context on the word which uniquely identifies the referent, albeit on the verb in the current study. Results on the noun, where uncertainty is reduced and expectations are generated in the current design, were mixed and were most likely influenced by design decisions specific to each experiment. These results show that processing of the reference-resolving word --- whether it be a noun or a verb --- reliably benefits from the prior linguistic and visual information that lead to the generation of concrete expectations.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

The results of a highly influential study that tested the predictions of the Rational Speech Act (RSA) model suggest that (a) listeners use pragmatic reasoning in one-shot web-based referential communication games despite the artificial, highly constrained, and minimally interactive nature of the task, and (b) that RSA accurately captures this behavior. In this work, we reevaluate the contribution of the pragmatic reasoning formalized by RSA in explaining listener behavior by comparing RSA to a baseline literal listener model that is only driven by literal word meaning and the prior probability of referring to an object. Across three experiments we observe only modest evidence of pragmatic behavior in one-shot web-based language games, and only under very limited circumstances. We find that although RSA provides a strong fit to listener responses, it does not perform better than the baseline literal listener model. Our results suggest that while participants playing the role of the Speaker are informative in these one-shot web-based reference games, participants playing the role of the Listener only rarely take this Speaker behavior into account to reason about the intended referent. In addition, we show that RSA’s fit is primarily due to a combination of non-pragmatic factors, perhaps the most surprising of which is that in the majority of conditions that are amenable to pragmatic reasoning, RSA (accurately) predicts that listeners will behave non-pragmatically. This leads us to conclude that RSA’s strong overall correlation with human behavior in one-shot web-based language games does not reflect listener’s pragmatic reasoning about informative speakers.

• https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0248388
• https://doi.org/10.1371/journal.pone.0248388

@article{Sikos2021,
title = {Reevaluating pragmatic reasoning in language games},
author = {Les Sikos and Noortje Venhuizen and Heiner Drenhaus and Matthew W. Crocker},
url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0248388},
doi = {https://doi.org/10.1371/journal.pone.0248388},
year = {2021},
date = {2021-03-17},
journal = {PLOS ONE},
abstract = {The results of a highly influential study that tested the predictions of the Rational Speech Act (RSA) model suggest that (a) listeners use pragmatic reasoning in one-shot web-based referential communication games despite the artificial, highly constrained, and minimally interactive nature of the task, and (b) that RSA accurately captures this behavior. In this work, we reevaluate the contribution of the pragmatic reasoning formalized by RSA in explaining listener behavior by comparing RSA to a baseline literal listener model that is only driven by literal word meaning and the prior probability of referring to an object. Across three experiments we observe only modest evidence of pragmatic behavior in one-shot web-based language games, and only under very limited circumstances. We find that although RSA provides a strong fit to listener responses, it does not perform better than the baseline literal listener model. Our results suggest that while participants playing the role of the Speaker are informative in these one-shot web-based reference games, participants playing the role of the Listener only rarely take this Speaker behavior into account to reason about the intended referent. In addition, we show that RSA’s fit is primarily due to a combination of non-pragmatic factors, perhaps the most surprising of which is that in the majority of conditions that are amenable to pragmatic reasoning, RSA (accurately) predicts that listeners will behave non-pragmatically. This leads us to conclude that RSA’s strong overall correlation with human behavior in one-shot web-based language games does not reflect listener’s pragmatic reasoning about informative speakers.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

This thesis presents results from three ERP experiments on the influence of speaker gaze on listeners’ sentence comprehension with focus on the utilization of speaker gaze as part of the communicative signal. The first two experiments investigated whether speaker gaze was utilized in situated communication to form expectations about upcoming referents in an unfolding sentence. Participants were presented with a face performing gaze actions toward three objects surrounding it time aligned to utterances that compared two of the three objects.

Participants were asked to judge whether the sentence they heard was true given the provided scene. Gaze cues preceded the naming of the corresponding object by 800ms. The gaze cue preceding the mentioning of the second object was manipulated such that it was either Congruent, Incongruent or Uninformative (Averted toward an empty position in experiment 1 and Mutual (redirected toward the listener) in Experiment 2). The results showed that speaker gaze was used to form expectations about the unfolding sentence indicated by three observed ERP components that index different underlying mechanisms of language comprehension: an increased Phonological Mapping Negativity (PMN) was observed when an unexpected (Incongruent) or unpredictable (Uninformative) phoneme is encountered. The retrieval of a referent’s semantics was indexed by an N400 effect in response to referents following both Incongruent and Uninformative gaze. Additionally, an increased P600 response was present only for preceding Incongruent gaze, indexing the revision process of the mental representation of the situation. The involvement of these mechanisms has been supported by the findings of the third experiment, in which linguistic content was presented to serve as a predictive cue for subsequent speaker gaze. In this experiment the sentence structure enabled participants to anticipate upcoming referents based on the preceding linguistic content. Thus, gaze cues preceding the mentioning of the referent could also be anticipated.

The results showed the involvement of the same mechanisms as in the first two experiments on the referent itself, only when preceding gaze was absent. In the presence of object-directed gaze, while there were no longer significant effects on the referent itself, effects of semantic retrieval (N400) and integration with sentence meaning (P3b) were found on the gaze cue. Effects in the P3b (Gaze) and P600 (Referent) time-window further provided support for the presence of a mechanism of monitoring of the mental representation of the situation that subsumes the integration into that representation: A positive deflection was found whenever the communicative signal completed the mental representation such that an evaluation of that representation was possible. Taken together, the results provide support for the view that speaker gaze, in situated communication, is interpreted as part of the communicative signal and incrementally used to inform the mental representation of the situation simultaneously with the linguistic signal and that the mental representation is utilized to generate expectations about upcoming referents in an unfolding utterance.

• DissertationJachmann     (3.44MB)
• http://nbn-resolving.de/urn:nbn:de:bsz:291--ds-313090
• https://doi.org/10.22028/D291-31309

@phdthesis{Jachmann2020,
title = {The immediate influence of speaker gaze on situated speech comprehension : evidence from multiple ERP components},
author = {Torsten Jachmann},
url = {http://nbn-resolving.de/urn:nbn:de:bsz:291--ds-313090},
doi = {https://doi.org/10.22028/D291-31309},
year = {2020},
date = {2020},
school = {Saarland University},
address = {Saarbruecken, Germany},
abstract = {This thesis presents results from three ERP experiments on the influence of speaker gaze on listeners’ sentence comprehension with focus on the utilization of speaker gaze as part of the communicative signal. The first two experiments investigated whether speaker gaze was utilized in situated communication to form expectations about upcoming referents in an unfolding sentence. Participants were presented with a face performing gaze actions toward three objects surrounding it time aligned to utterances that compared two of the three objects. Participants were asked to judge whether the sentence they heard was true given the provided scene. Gaze cues preceded the naming of the corresponding object by 800ms. The gaze cue preceding the mentioning of the second object was manipulated such that it was either Congruent, Incongruent or Uninformative (Averted toward an empty position in experiment 1 and Mutual (redirected toward the listener) in Experiment 2). The results showed that speaker gaze was used to form expectations about the unfolding sentence indicated by three observed ERP components that index different underlying mechanisms of language comprehension: an increased Phonological Mapping Negativity (PMN) was observed when an unexpected (Incongruent) or unpredictable (Uninformative) phoneme is encountered. The retrieval of a referent’s semantics was indexed by an N400 effect in response to referents following both Incongruent and Uninformative gaze. Additionally, an increased P600 response was present only for preceding Incongruent gaze, indexing the revision process of the mental representation of the situation. The involvement of these mechanisms has been supported by the findings of the third experiment, in which linguistic content was presented to serve as a predictive cue for subsequent speaker gaze. In this experiment the sentence structure enabled participants to anticipate upcoming referents based on the preceding linguistic content. Thus, gaze cues preceding the mentioning of the referent could also be anticipated. The results showed the involvement of the same mechanisms as in the first two experiments on the referent itself, only when preceding gaze was absent. In the presence of object-directed gaze, while there were no longer significant effects on the referent itself, effects of semantic retrieval (N400) and integration with sentence meaning (P3b) were found on the gaze cue. Effects in the P3b (Gaze) and P600 (Referent) time-window further provided support for the presence of a mechanism of monitoring of the mental representation of the situation that subsumes the integration into that representation: A positive deflection was found whenever the communicative signal completed the mental representation such that an evaluation of that representation was possible. Taken together, the results provide support for the view that speaker gaze, in situated communication, is interpreted as part of the communicative signal and incrementally used to inform the mental representation of the situation simultaneously with the linguistic signal and that the mental representation is utilized to generate expectations about upcoming referents in an unfolding utterance.},
pubstate = {published},
type = {phdthesis}
}

Copy BibTeX to Clipboard

Contrary to the Gricean maxims of Quantity (Grice, 1975), it has been repeatedly shown that speakers often include redundant information in their utterances (over- specifications). Previous research on referential communication has long debated whether this redundancy is the result of speaker-internal or addressee-oriented processes, while it is also unclear whether referential redundancy hinders or facilitates comprehension. We present a bounded-rational account of referential redundancy, according to which any word in an utterance, even if it is redundant, can be beneficial to comprehension, to the extent that it facilitates the reduction of listeners’ uncertainty regarding the target referent in a co-present visual scene. Information-theoretic metrics, such as Shannon’s entropy (Shannon, 1948), were employed in order to quantify this uncertainty in bits of information, and gain an estimate of the cognitive effort related to referential processing. Under this account, speakers may, therefore, utilise redundant adjectives in order to reduce the visually-determined entropy (and thereby their listeners’ cognitive effort) more uniformly across their utterances. In a series of experiments, we examined both the comprehension and the production of over-specifications in complex visual contexts. Our findings are in line with the bounded-rational account. Specifically, we present evidence that: (a) in view of complex visual scenes, listeners’ processing and identification of the target referent may be facilitated by the use of redundant adjectives, as well as by a more uniform reduction of uncertainty across the utterance, and (b) that, while both speaker-internal and addressee-oriented processes are at play in the production of over-specifications, listeners’ processing concerns may also influence the encoding of redundant adjectives, at least for some speakers, who encode redundant adjectives more frequently when these adjectives contribute to a more uniform reduction of referential entropy.

• https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/29453
• https://doi.org/10.22028/D291-31436

@phdthesis{Tourtouri2020,
title = {Rational redundancy in situated communication},
author = {Elli Tourtouri},
url = {https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/29453},
doi = {https://doi.org/10.22028/D291-31436},
year = {2020},
date = {2020},
school = {Saarland University},
address = {Saarbr{\"u}cken},
abstract = {Contrary to the Gricean maxims of Quantity (Grice, 1975), it has been repeatedly shown that speakers often include redundant information in their utterances (over- specifications). Previous research on referential communication has long debated whether this redundancy is the result of speaker-internal or addressee-oriented processes, while it is also unclear whether referential redundancy hinders or facilitates comprehension. We present a bounded-rational account of referential redundancy, according to which any word in an utterance, even if it is redundant, can be beneficial to comprehension, to the extent that it facilitates the reduction of listeners’ uncertainty regarding the target referent in a co-present visual scene. Information-theoretic metrics, such as Shannon’s entropy (Shannon, 1948), were employed in order to quantify this uncertainty in bits of information, and gain an estimate of the cognitive effort related to referential processing. Under this account, speakers may, therefore, utilise redundant adjectives in order to reduce the visually-determined entropy (and thereby their listeners’ cognitive effort) more uniformly across their utterances. In a series of experiments, we examined both the comprehension and the production of over-specifications in complex visual contexts. Our findings are in line with the bounded-rational account. Specifically, we present evidence that: (a) in view of complex visual scenes, listeners’ processing and identification of the target referent may be facilitated by the use of redundant adjectives, as well as by a more uniform reduction of uncertainty across the utterance, and (b) that, while both speaker-internal and addressee-oriented processes are at play in the production of over-specifications, listeners’ processing concerns may also influence the encoding of redundant adjectives, at least for some speakers, who encode redundant adjectives more frequently when these adjectives contribute to a more uniform reduction of referential entropy.},
pubstate = {published},
type = {phdthesis}
}

Copy BibTeX to Clipboard

This dissertation approaches the task of modeling human sentence production from a connectionist point of view, and using distributed semantic representations. The main questions it tries to address are: (i) whether the distributed semantic representations defined by Frank et al. (2009) are suitable to model sentence production using artificial neural networks, (ii) the behavior and internal mechanism of a model that uses this representations and recurrent neural networks, and (iii) a mechanistic account of the Uniform Information Density Hypothesis (UID; Jaeger, 2006; Levy and Jaeger, 2007). Regarding the first point, the semantic representations of Frank et al. (2009), called situation vectors are points in a vector space where each vector contains information about the observations in which an event and a corresponding sentence are true. These representations have been successfully used to model language comprehension (e.g., Frank et al., 2009; Venhuizen et al., 2018). During the construction of these vectors, however, a dimensionality reduction process introduces some loss of information, which causes some aspects to be no longer recognizable, reducing the performance of a model that utilizes them. In order to address this issue, belief vectors are introduced, which could be regarded as an alternative way to obtain semantic representations of manageable dimensionality. These two types of representations (situation and belief vectors) are evaluated using them as input for a sentence production model that implements an extension of a Simple Recurrent Neural network (Elman, 1990). This model was tested under different conditions corresponding to different levels of systematicity, which is the ability of a model to generalize from a set of known items to a set of novel ones. Systematicity is an essential attribute that a model of sentence processing has to possess, considering that the number of sentences that can be generated for a given language is infinite, and therefore it is not feasible to memorize all possible message-sentence pairs. The results showed that the model was able to generalize with a very high performance in all test conditions, demonstrating a systematic behavior. Furthermore, the errors that it elicited were related to very similar semantic representations, reflecting the speech error literature, which states that speech errors involve elements with semantic or phonological similarity. This result further demonstrates the systematic behavior of the model, as it processes similar semantic representations in a similar way, even if they are new to the model. Regarding the second point, the sentence production model was analyzed in two different ways. First, by looking at the sentences it produces, including the errors elicited, highlighting difficulties and preferences of the model. The results revealed that the model learns the syntactic patterns of the language, reflecting its statistical nature, and that its main difficulty is related to very similar semantic representations, sometimes producing unintended sentences that are however very semantically related to the intended ones. Second, the connection weights and activation patterns of the model were also analyzed, reaching an algorithmic account of the internal processing of the model. According to this, the input semantic representation activates the words that are related to its content, giving an idea of their order by providing relatively more activation to words that are likely to appear early in the sentence. Then, at each time step the word that was previously produced activates syntactic and semantic constraints on the next word productions, while the context units of the recurrence preserve information through time, allowing the model to enforce long distance dependencies. We propose that these results can inform about the internal processing of models with similar architecture. Regarding the third point, an extension of the model is proposed with the goal of modeling UID. According to UID, language production is an efficient process affected by a tendency to produce linguistic units distributing the information as uniformly as possible and close to the capacity of the communication channel, given the encoding possibilities of the language, thus optimizing the amount of information that is transmitted per time unit. This extension of the model approaches UID by balancing two different production strategies: one where the model produces the word with highest probability given the semantics and the previously produced words, and another one where the model produces the word that would minimize the sentence length given the semantic representation and the previously produced words. By combining these two strategies, the model was able to produce sentences with different levels of information density and uniformity, providing a first step to model UID at the algorithmic level of analysis. In sum, the results show that the distributed semantic representations of Frank et al. (2009) can be used to model sentence production, exhibiting systematicity. Moreover, an algorithmic account of the internal behavior of the model was reached, with the potential to generalize to other models with similar architecture. Finally, a model of UID is presented, highlighting some important aspects about UID that need to be addressed in order to go from the formulation of UID at the computational level of analysis to a mechanistic account at the algorithmic level.

• http://nbn-resolving.de/urn:nbn:de:bsz:291--ds-279340
• https://doi.org/http://dx.doi.org/10.22028/D291-27934

@phdthesis{Calvillo_diss_2019,
title = {Connectionist language production : distributed representations and the uniform information density hypothesis},
author = {Jesús Calvillo},
url = {http://nbn-resolving.de/urn:nbn:de:bsz:291--ds-279340},
doi = {https://doi.org/http://dx.doi.org/10.22028/D291-27934},
year = {2019},
date = {2019},
school = {Saarland University},
address = {Saarbruecken, Germany},
abstract = {This dissertation approaches the task of modeling human sentence production from a connectionist point of view, and using distributed semantic representations. The main questions it tries to address are: (i) whether the distributed semantic representations defined by Frank et al. (2009) are suitable to model sentence production using artificial neural networks, (ii) the behavior and internal mechanism of a model that uses this representations and recurrent neural networks, and (iii) a mechanistic account of the Uniform Information Density Hypothesis (UID; Jaeger, 2006; Levy and Jaeger, 2007). Regarding the first point, the semantic representations of Frank et al. (2009), called situation vectors are points in a vector space where each vector contains information about the observations in which an event and a corresponding sentence are true. These representations have been successfully used to model language comprehension (e.g., Frank et al., 2009; Venhuizen et al., 2018). During the construction of these vectors, however, a dimensionality reduction process introduces some loss of information, which causes some aspects to be no longer recognizable, reducing the performance of a model that utilizes them. In order to address this issue, belief vectors are introduced, which could be regarded as an alternative way to obtain semantic representations of manageable dimensionality. These two types of representations (situation and belief vectors) are evaluated using them as input for a sentence production model that implements an extension of a Simple Recurrent Neural network (Elman, 1990). This model was tested under different conditions corresponding to different levels of systematicity, which is the ability of a model to generalize from a set of known items to a set of novel ones. Systematicity is an essential attribute that a model of sentence processing has to possess, considering that the number of sentences that can be generated for a given language is infinite, and therefore it is not feasible to memorize all possible message-sentence pairs. The results showed that the model was able to generalize with a very high performance in all test conditions, demonstrating a systematic behavior. Furthermore, the errors that it elicited were related to very similar semantic representations, reflecting the speech error literature, which states that speech errors involve elements with semantic or phonological similarity. This result further demonstrates the systematic behavior of the model, as it processes similar semantic representations in a similar way, even if they are new to the model. Regarding the second point, the sentence production model was analyzed in two different ways. First, by looking at the sentences it produces, including the errors elicited, highlighting difficulties and preferences of the model. The results revealed that the model learns the syntactic patterns of the language, reflecting its statistical nature, and that its main difficulty is related to very similar semantic representations, sometimes producing unintended sentences that are however very semantically related to the intended ones. Second, the connection weights and activation patterns of the model were also analyzed, reaching an algorithmic account of the internal processing of the model. According to this, the input semantic representation activates the words that are related to its content, giving an idea of their order by providing relatively more activation to words that are likely to appear early in the sentence. Then, at each time step the word that was previously produced activates syntactic and semantic constraints on the next word productions, while the context units of the recurrence preserve information through time, allowing the model to enforce long distance dependencies. We propose that these results can inform about the internal processing of models with similar architecture. Regarding the third point, an extension of the model is proposed with the goal of modeling UID. According to UID, language production is an efficient process affected by a tendency to produce linguistic units distributing the information as uniformly as possible and close to the capacity of the communication channel, given the encoding possibilities of the language, thus optimizing the amount of information that is transmitted per time unit. This extension of the model approaches UID by balancing two different production strategies: one where the model produces the word with highest probability given the semantics and the previously produced words, and another one where the model produces the word that would minimize the sentence length given the semantic representation and the previously produced words. By combining these two strategies, the model was able to produce sentences with different levels of information density and uniformity, providing a first step to model UID at the algorithmic level of analysis. In sum, the results show that the distributed semantic representations of Frank et al. (2009) can be used to model sentence production, exhibiting systematicity. Moreover, an algorithmic account of the internal behavior of the model was reached, with the potential to generalize to other models with similar architecture. Finally, a model of UID is presented, highlighting some important aspects about UID that need to be addressed in order to go from the formulation of UID at the computational level of analysis to a mechanistic account at the algorithmic level.},
pubstate = {published},
type = {phdthesis}
}

Copy BibTeX to Clipboard

Behavioral studies have shown that speaker gaze to objects in a co-present scene can influence listeners’ sentence comprehension. To gain deeper insight into the mechanisms involved in gaze processing and integration, we conducted two ERP experiments (N = 30, Age: [18, 32] and [19, 33] respectively). Participants watched a centrally positioned face performing gaze actions aligned to utterances comparing two out of three displayed objects. They were asked to judge whether the sentence was true given the provided scene. We manipulated the second gaze cue to be either Congruent (baseline), Incongruent or Averted (Exp1)/Mutual (Exp2). When speaker gaze is used to form lexical expectations about upcoming referents, we found an attenuated N200 when phonological information confirms these expectations (Congruent). Similarly, we observed attenuated N400 amplitudes when gaze-cued expectations (Congruent) facilitate lexical retrieval. Crucially, only a violation of gaze-cued lexical expectations (Incongruent) leads to a P600 effect, suggesting the necessity to revise the mental representation of the situation. Our results support the hypothesis that gaze is utilized above and beyond simply enhancing a cued object’s prominence. Rather, gaze to objects leads to their integration into the mental representation of the situation before they are mentioned.

• https://www.sciencedirect.com/science/article/pii/S0278262619300120
• https://doi.org/10.1016/j.bandc.2019.05.009

@article{Jachmann2019b,
title = {Influence of speakers’ gaze on situated language comprehension: Evidence from Event-Related Potentials},
author = {Torsten Jachmann and Heiner Drenhaus and Maria Staudte and Matthew W. Crocker},
url = {https://www.sciencedirect.com/science/article/pii/S0278262619300120},
doi = {https://doi.org/10.1016/j.bandc.2019.05.009},
year = {2019},
date = {2019},
journal = {Brain and cognition},
pages = {103571},
publisher = {Elsevier},
volume = {135},
abstract = {Behavioral studies have shown that speaker gaze to objects in a co-present scene can influence listeners’ sentence comprehension. To gain deeper insight into the mechanisms involved in gaze processing and integration, we conducted two ERP experiments (N = 30, Age: [18, 32] and [19, 33] respectively). Participants watched a centrally positioned face performing gaze actions aligned to utterances comparing two out of three displayed objects. They were asked to judge whether the sentence was true given the provided scene. We manipulated the second gaze cue to be either Congruent (baseline), Incongruent or Averted (Exp1)/Mutual (Exp2). When speaker gaze is used to form lexical expectations about upcoming referents, we found an attenuated N200 when phonological information confirms these expectations (Congruent). Similarly, we observed attenuated N400 amplitudes when gaze-cued expectations (Congruent) facilitate lexical retrieval. Crucially, only a violation of gaze-cued lexical expectations (Incongruent) leads to a P600 effect, suggesting the necessity to revise the mental representation of the situation. Our results support the hypothesis that gaze is utilized above and beyond simply enhancing a cued object’s prominence. Rather, gaze to objects leads to their integration into the mental representation of the situation before they are mentioned.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

The processing difficulty of each word we encounter in a sentence is affected by both our prior linguistic experience and our general knowledge about the world. Computational models of incremental language processing have, however, been limited in accounting for the influence of world knowledge.

We develop an incremental model of language comprehension that constructs – on a word-by-word basis – rich, probabilistic situation model representations. To quantify linguistic processing effort, we adopt Surprisal Theory, which asserts that the processing difficulty incurred by a word is inversely proportional to its expectancy (Hale, 2001; Levy, 2008). In contrast with typical language model implementations of surprisal, the proposed model instantiates a novel comprehension-centric metric of surprisal that reflects the likelihood of the unfolding utterance meaning as established after processing each word. Simulations are presented that demonstrate that linguistic experience and world knowledge are integrated in the model at the level of interpretation and combine in determining online expectations.

• Venhuizen2019ExpectationComprehension     (0.97MB)
• https://www.tandfonline.com/doi/full/10.1080/0163853X.2018.1448677
• https://doi.org/10.1080/0163853X.2018.1448677

@article{Venhuizen2019,
title = {Expectation-based Comprehension: Modeling the interaction of world knowledge and linguistic experience},
author = {Noortje Venhuizen and Matthew W. Crocker and Harm Brouwer},
url = {https://www.tandfonline.com/doi/full/10.1080/0163853X.2018.1448677},
doi = {https://doi.org/10.1080/0163853X.2018.1448677},
year = {2019},
date = {2019},
journal = {Discourse Processes},
pages = {229-255},
volume = {56},
number = {3},
abstract = {The processing difficulty of each word we encounter in a sentence is affected by both our prior linguistic experience and our general knowledge about the world. Computational models of incremental language processing have, however, been limited in accounting for the influence of world knowledge. We develop an incremental model of language comprehension that constructs - on a word-by-word basis - rich, probabilistic situation model representations. To quantify linguistic processing effort, we adopt Surprisal Theory, which asserts that the processing difficulty incurred by a word is inversely proportional to its expectancy (Hale, 2001; Levy, 2008). In contrast with typical language model implementations of surprisal, the proposed model instantiates a novel comprehension-centric metric of surprisal that reflects the likelihood of the unfolding utterance meaning as established after processing each word. Simulations are presented that demonstrate that linguistic experience and world knowledge are integrated in the model at the level of interpretation and combine in determining online expectations.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

Contrary to the Gricean maxims of quantity (Grice, in: Cole, Morgan (eds) Syntax and semantics: speech acts, vol III, pp 41-58, Academic Press, New York, 1975), it has been repeatedly shown that speakers often include redundant information in their utterances (over-specifications). Previous research on referential communication has long debated whether this redundancy is the result of speaker-internal or addressee-oriented processes, while it is also unclear whether referential redundancy hinders or facilitates comprehension.

We present an information-theoretic explanation for the use of over-specification in visually-situated communication, which quantifies the amount of uncertainty regarding the referent as entropy (Shannon in Bell Syst Tech J 5:10, https://doi.org/10.1002/j.1538-7305.1948.tb01338.x, 1948). Examining both the comprehension and production of over-specifications, we present evidence that (a) listeners’ processing is facilitated by the use of redundancy as well as by a greater reduction of uncertainty early on in the utterance, and (b) that at least for some speakers, listeners’ processing concerns influence their encoding of over-specifications: Speakers were more likely to use redundant adjectives when these adjectives reduced entropy to a higher degree than adjectives necessary for target identification.

• Rational-over-specification-in-visually-situated-comprehension-and-production     (1.72MB)
• https://link.springer.com/article/10.1007%2Fs41809-019-00032-6
• https://doi.org/10.1007/s41809-019-00032-6

@article{Tourtouri2019,
title = {Rational over-specification in visually-situated comprehension and production},
author = {Elli Tourtouri and Francesca Delogu and Les Sikos and Matthew W. Crocker},
url = {https://link.springer.com/article/10.1007%2Fs41809-019-00032-6},
doi = {https://doi.org/10.1007/s41809-019-00032-6},
year = {2019},
date = {2019},
journal = {Journal of Cultural Cognitive Science},
pages = {175-202},
volume = {3},
number = {2},
abstract = {Contrary to the Gricean maxims of quantity (Grice, in: Cole, Morgan (eds) Syntax and semantics: speech acts, vol III, pp 41-58, Academic Press, New York, 1975), it has been repeatedly shown that speakers often include redundant information in their utterances (over-specifications). Previous research on referential communication has long debated whether this redundancy is the result of speaker-internal or addressee-oriented processes, while it is also unclear whether referential redundancy hinders or facilitates comprehension. We present an information-theoretic explanation for the use of over-specification in visually-situated communication, which quantifies the amount of uncertainty regarding the referent as entropy (Shannon in Bell Syst Tech J 5:10, https://doi.org/10.1002/j.1538-7305.1948.tb01338.x, 1948). Examining both the comprehension and production of over-specifications, we present evidence that (a) listeners’ processing is facilitated by the use of redundancy as well as by a greater reduction of uncertainty early on in the utterance, and (b) that at least for some speakers, listeners’ processing concerns influence their encoding of over-specifications: Speakers were more likely to use redundant adjectives when these adjectives reduced entropy to a higher degree than adjectives necessary for target identification.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

Specificity in referential communication

• Grice’s Maxim of Quantity [1]: Speakers should produce only informa9on that is strictly necessary for identifying the target
• However, it is possible to establish reference with either minimally-specified (MS; precise) or over-specified (OS; redundant) expressions
• Moreover, speakers overspecify frequently and systematically [e.g., 2-6]

Q: Why do people overspecificy?

• Tourtouri_18_CUNY_poster     (0.34MB)
• https://www.researchgate.net/publication/323809271_Referential_entropy_influences_overspecification_Evidence_from_production

@miscellaneous{Tourtourietal2018a,
title = {Referential Entropy influences Overspecification: Evidence from Production},
author = {Elli Tourtouri and Les Sikos and Matthew W. Crocker},
url = {https://www.researchgate.net/publication/323809271_Referential_entropy_influences_overspecification_Evidence_from_production},
year = {2018},
date = {2018},
booktitle = {31st Annual CUNY Sentence Processing Conference},
publisher = {UC Davis},
address = {Davis CA, USA},
abstract = {Specificity in referential communication

• Grice’s Maxim of Quantity [1]: Speakers should produce only informa9on that is strictly necessary for identifying the target
• However, it is possible to establish reference with either minimally-specified (MS; precise) or over-specified (OS; redundant) expressions
• Moreover, speakers overspecify frequently and systematically [e.g., 2-6]

Copy BibTeX to Clipboard

Gaze Cues in face-to-face interactions

• Speakers‘ direct their gaze toward an object approximately 800ms before mentioning. (Griffin & Bock, 2000)
• Previous studies showed that listeners utilize speakers‘ gaze to form predictions about the unfolding sentence. (Jachmann et al., 2017)
• Do listeners utilize this external cue to validate expectations about the unfolding sentence? And, if so, how does this effect the comprehension of the noun?

• https://www.researchgate.net/publication/327623334_DISCONFIRMATION_OF_LINGUISTIC_PREDICTION_BY_NON-LINGUISTIC_CUES

@miscellaneous{Jachmann2018,
title = {(Dis-)confirmation of linguistic prediction by non-linguistic cues},
author = {Torsten Jachmann and Heiner Drenhaus and Maria Staudte and Matthew W. Crocker},
url = {https://www.researchgate.net/publication/327623334_DISCONFIRMATION_OF_LINGUISTIC_PREDICTION_BY_NON-LINGUISTIC_CUES},
year = {2018},
date = {2018},
booktitle = {24th Annual Conference on Architectures and Mechanisms for Language Processing (AMLaP)},
address = {Berlin},
abstract = {Gaze Cues in face-to-face interactions

• Speakers‘ direct their gaze toward an object approximately 800ms before mentioning. (Griffin & Bock, 2000)
• Previous studies showed that listeners utilize speakers‘ gaze to form predictions about the unfolding sentence. (Jachmann et al., 2017)
• Do listeners utilize this external cue to validate expectations about the unfolding sentence? And, if so, how does this effect the comprehension of the noun?

Copy BibTeX to Clipboard

When reading a text describing an everyday activity, comprehenders build a model of the situation described that includes prior knowledge of the entities, locations, and sequences of actions that typically occur within the event. Previous work has demonstrated that such knowledge guides the processing of incoming information by making event boundaries more or less expected.

In the present ERP study, we investigated whether comprehenders’ expectations about event boundaries are influenced by how elaborately common events are described in the context. Participants read short stories in which a common activity (e.g., washing the dishes) was described either in brief or in an elaborate manner. The final sentence contained a target word referring to a more predictable action marking a fine event boundary (e.g., drying) or a less predictable action, marking a coarse event boundary (e.g., jogging). The results revealed a larger N400 effect for coarse event boundaries compared to fine event boundaries, but no interaction with description length. Between 600 and 1000 ms, however, elaborate contexts elicited a larger frontal positivity compared to brief contexts.

This effect was largely driven by less predictable targets, marking coarse event boundaries. We interpret the P600 effect as indexing the updating of the situation model at event boundaries, consistent with Event Segmentation Theory (EST). The updating process is more demanding with coarse event boundaries, which presumably require the construction of a new situation model. Electronic supplementary material The online version of this article (10.3758/s13421-017-0766-4) contains supplementary material, which is available to authorized users.

• On-the-predictability-of-event-boundaries-in-discourse     (0.73MB)
• https://www.researchgate.net/publication/321175597_On_the_predictability_of_event_boundaries_in_discourse_An_ERP_investigation
• https://doi.org/https://doi.org/10.3758/s13421-017-0766-4

@article{Delogu2018,
title = {On the predictability of event-boundaries in discourse: an ERP investigation},
author = {Francesca Delogu and Heiner Drenhaus and Matthew W. Crocker},
url = {https://www.researchgate.net/publication/321175597_On_the_predictability_of_event_boundaries_in_discourse_An_ERP_investigation},
doi = {https://doi.org/https://doi.org/10.3758/s13421-017-0766-4},
year = {2018},
date = {2018},
journal = {Memory and Cognition},
pages = {315-325},
volume = {46},
number = {2},
abstract = {When reading a text describing an everyday activity, comprehenders build a model of the situation described that includes prior knowledge of the entities, locations, and sequences of actions that typically occur within the event. Previous work has demonstrated that such knowledge guides the processing of incoming information by making event boundaries more or less expected. In the present ERP study, we investigated whether comprehenders’ expectations about event boundaries are influenced by how elaborately common events are described in the context. Participants read short stories in which a common activity (e.g., washing the dishes) was described either in brief or in an elaborate manner. The final sentence contained a target word referring to a more predictable action marking a fine event boundary (e.g., drying) or a less predictable action, marking a coarse event boundary (e.g., jogging). The results revealed a larger N400 effect for coarse event boundaries compared to fine event boundaries, but no interaction with description length. Between 600 and 1000 ms, however, elaborate contexts elicited a larger frontal positivity compared to brief contexts. This effect was largely driven by less predictable targets, marking coarse event boundaries. We interpret the P600 effect as indexing the updating of the situation model at event boundaries, consistent with Event Segmentation Theory (EST). The updating process is more demanding with coarse event boundaries, which presumably require the construction of a new situation model. Electronic supplementary material The online version of this article (10.3758/s13421-017-0766-4) contains supplementary material, which is available to authorized users.},
pubstate = {published},
type = {article}
}

Copy BibTeX to Clipboard

A word’s predictability or surprisal, as determined by cloze probabilities or language models (e.g. Frank, Otten, Galli, & Vigliocco, 2015) is related to processing effort, in that less expected words take more effort to process (e.g. Hale, 2001). A words surprisal, however, may also be influenced by the non-linguistic context, such as visual cues: In the visual world paradigm (VWP), for example, anticipatory eye movements suggest that comprehenders exploit the scene to predict what will be mentioned next (Altmann & Kamide, 1999).

How visual context affects word surprisal and processing effort, however, remains unclear. Here, we present evidence that visually-determined probabilistic expectations for a spoken target word predict graded processing effort for that word, in both pupillometric (ICA) and ERP (N400) measures. These findings demonstrate that the non-linguistic context can immediately influence both lexical expectations, and surprisal-based processing effort.

• https://www.researchgate.net/publication/325644935_Multimodal_Surprisal_in_the_N400_and_the_Index_of_Cognitive_Activity

@inproceedings{Ankener2018,
title = {Multimodal Surprisal in the N400 and the Index of Cognitive Activity},
author = {Christine Ankener and Heiner Drenhaus and Matthew W. Crocker and Maria Staudte},
url = {https://www.researchgate.net/publication/325644935_Multimodal_Surprisal_in_the_N400_and_the_Index_of_Cognitive_Activity},
year = {2018},
date = {2018},
booktitle = {Proceedings of the 40th Annual Cognitive Science Society Meeting},
pages = {94-100},
publisher = {The Cognitive Science Society},
address = {Madison, Wisconsin},
abstract = {A word’s predictability or surprisal, as determined by cloze probabilities or language models (e.g. Frank, Otten, Galli, & Vigliocco, 2015) is related to processing effort, in that less expected words take more effort to process (e.g. Hale, 2001). A words surprisal, however, may also be influenced by the non-linguistic context, such as visual cues: In the visual world paradigm (VWP), for example, anticipatory eye movements suggest that comprehenders exploit the scene to predict what will be mentioned next (Altmann & Kamide, 1999). How visual context affects word surprisal and processing effort, however, remains unclear. Here, we present evidence that visually-determined probabilistic expectations for a spoken target word predict graded processing effort for that word, in both pupillometric (ICA) and ERP (N400) measures. These findings demonstrate that the non-linguistic context can immediately influence both lexical expectations, and surprisal-based processing effort.},
pubstate = {published},
type = {inproceedings}
}

Copy BibTeX to Clipboard