@phdthesis{Jachmann2020,
title = {The immediate influence of speaker gaze on situated speech comprehension : evidence from multiple ERP components},
author = {Torsten Jachmann},
url = {http://nbn-resolving.de/urn:nbn:de:bsz:291--ds-313090},
doi = {https://doi.org/10.22028/D291-31309},
year = {2020},
date = {2020},
school  = {Saarland University},
address = {Saarbruecken, Germany},
abstract = {This thesis presents results from three ERP experiments on the influence of speaker gaze on listeners’ sentence comprehension with focus on the utilization of speaker gaze as part of the communicative signal. The first two experiments investigated whether speaker gaze was utilized in situated communication to form expectations about upcoming referents in an unfolding sentence. Participants were presented with a face performing gaze actions toward three objects surrounding it time aligned to utterances that compared two of the three objects.

Participants were asked to judge whether the sentence they heard was true given the provided scene. Gaze cues preceded the naming of the corresponding object by 800ms. The gaze cue preceding the mentioning of the second object was manipulated such that it was either Congruent, Incongruent or Uninformative (Averted toward an empty position in experiment 1 and Mutual (redirected toward the listener) in Experiment 2). The results showed that speaker gaze was used to form expectations about the unfolding sentence indicated by three observed ERP components that index different underlying mechanisms of language comprehension: an increased Phonological Mapping Negativity (PMN) was observed when an unexpected (Incongruent) or unpredictable (Uninformative) phoneme is encountered. The retrieval of a referent’s semantics was indexed by an N400 effect in response to referents following both Incongruent and Uninformative gaze. Additionally, an increased P600 response was present only for preceding Incongruent gaze, indexing the revision process of the mental representation of the situation. The involvement of these mechanisms has been supported by the findings of the third experiment, in which linguistic content was presented to serve as a predictive cue for subsequent speaker gaze. In this experiment the sentence structure enabled participants to anticipate upcoming referents based on the preceding linguistic content. Thus, gaze cues preceding the mentioning of the referent could also be anticipated.

The results showed the involvement of the same mechanisms as in the first two experiments on the referent itself, only when preceding gaze was absent. In the presence of object-directed gaze, while there were no longer significant effects on the referent itself, effects of semantic retrieval (N400) and integration with sentence meaning (P3b) were found on the gaze cue. Effects in the P3b (Gaze) and P600 (Referent) time-window further provided support for the presence of a mechanism of monitoring of the mental representation of the situation that subsumes the integration into that representation: A positive deflection was found whenever the communicative signal completed the mental representation such that an evaluation of that representation was possible. Taken together, the results provide support for the view that speaker gaze, in situated communication, is interpreted as part of the communicative signal and incrementally used to inform the mental representation of the situation simultaneously with the linguistic signal and that the mental representation is utilized to generate expectations about upcoming referents in an unfolding utterance.},
pubstate = {published},
type = {phdthesis}
}