From 32185a56d97bb787911a89e398f50e62aca16ae5 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 18 Nov 2026 09:47:37 +0100 Subject: [PATCH] Issue #6383: Switch more code to the central XML parser factories - Parse TEI (legacy) documents using the shared XML parser configuration from XmlParserUtils - Validate Perseus documents using the shared XML parser configuration before passing them to the reader - Add tests for the TEI (legacy) and Perseus readers Assisted-by: Anthropic Claude (claude-opus-5-5) --- inception/inception-io-perseus/pom.xml | 9 ++ .../io/perseus/PerseusFormatSupport.java | 34 +++++++ .../io/perseus/PerseusFormatSupportTest.java | 77 ++++++++++++++++ .../io/tei/dkprobackport/TeiReader.java | 10 +- .../io/tei/dkprobackport/TeiReaderTest.java | 92 +++++++++++++++++++ 5 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 inception/inception-io-perseus/src/test/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupportTest.java create mode 100644 inception/inception-io-tei/src/test/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReaderTest.java diff --git a/inception/inception-io-perseus/pom.xml b/inception/inception-io-perseus/pom.xml index 344887fcb83..33224c52c7b 100644 --- a/inception/inception-io-perseus/pom.xml +++ b/inception/inception-io-perseus/pom.xml @@ -38,6 +38,11 @@ inception-model ${project.version} + + de.tudarmstadt.ukp.inception.app + inception-support-xml + ${project.version} + org.springframework @@ -61,5 +66,9 @@ org.dkpro.core dkpro-core-io-perseus-asl + + org.dkpro.core + dkpro-core-api-resources-asl + \ No newline at end of file diff --git a/inception/inception-io-perseus/src/main/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupport.java b/inception/inception-io-perseus/src/main/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupport.java index 17152e68daf..22808af71ba 100644 --- a/inception/inception-io-perseus/src/main/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupport.java +++ b/inception/inception-io-perseus/src/main/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupport.java @@ -17,15 +17,28 @@ */ package de.tudarmstadt.ukp.inception.io.perseus; +import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.newSaxParser; import static org.apache.uima.fit.factory.CollectionReaderFactory.createReaderDescription; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.uima.cas.CAS; +import org.apache.uima.collection.CollectionException; import org.apache.uima.collection.CollectionReaderDescription; import org.apache.uima.resource.ResourceInitializationException; import org.apache.uima.resource.metadata.TypeSystemDescription; +import org.dkpro.core.api.resources.CompressionUtils; import org.dkpro.core.io.perseus.PerseusReader; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; import de.tudarmstadt.ukp.clarin.webanno.api.format.UimaReaderWriterFormatSupport_ImplBase; import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.io.perseus.config.PerseusFormatAutoConfiguration; /** @@ -71,4 +84,25 @@ public CollectionReaderDescription getReaderDescription(Project aProject, { return createReaderDescription(PerseusReader.class, aTSD); } + + @Override + public void read(SourceDocument aDocument, CAS aCas, File aFile) + throws ResourceInitializationException, IOException, CollectionException + { + validateXml(aFile); + + super.read(aDocument, aCas, aFile); + } + + static void validateXml(File aFile) throws IOException + { + try (var is = CompressionUtils.getInputStream(aFile.getName(), + Files.newInputStream(aFile.toPath()))) { + newSaxParser().parse(is, new DefaultHandler()); + } + catch (ParserConfigurationException | SAXException e) { + throw new IOException("Unable to parse [" + aFile.getName() + + "] - note that DOCTYPE declarations are not supported", e); + } + } } diff --git a/inception/inception-io-perseus/src/test/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupportTest.java b/inception/inception-io-perseus/src/test/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupportTest.java new file mode 100644 index 00000000000..483f319b43d --- /dev/null +++ b/inception/inception-io-perseus/src/test/java/de/tudarmstadt/ukp/inception/io/perseus/PerseusFormatSupportTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.io.perseus; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.uima.fit.factory.CasFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; + +class PerseusFormatSupportTest +{ + private static final String BODY = """ + + + + + + """; + + private @TempDir Path tempDir; + + @Test + void thatPlainFileIsRead() throws Exception + { + var file = tempDir.resolve("doc.xml"); + Files.writeString(file, BODY.formatted("-hello-"), UTF_8); + + var cas = CasFactory.createCas(); + new PerseusFormatSupport().read(new SourceDocument(), cas, file.toFile()); + + assertThat(cas.getDocumentText()).contains("A-hello-B"); + } + + @Test + void thatDoctypeIsRejected() throws Exception + { + var dtdFile = tempDir.resolve("doc.dtd"); + Files.writeString(dtdFile, "", UTF_8); + + var file = tempDir.resolve("doc.xml"); + Files.writeString(file, """ + + %p; ]> + """.replace("%s", dtdFile.toUri().toString()) + BODY.formatted("&ent;"), UTF_8); + + var cas = CasFactory.createCas(); + assertThatThrownBy( + () -> new PerseusFormatSupport().read(new SourceDocument(), cas, file.toFile())) // + .isInstanceOf(IOException.class) // + .hasMessageContaining("DOCTYPE"); + assertThat(cas.getDocumentText()).isNull(); + } +} diff --git a/inception/inception-io-tei/src/main/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReader.java b/inception/inception-io-tei/src/main/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReader.java index e964fd8a01a..c75bfbbd76a 100644 --- a/inception/inception-io-tei/src/main/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReader.java +++ b/inception/inception-io-tei/src/main/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReader.java @@ -33,6 +33,9 @@ import static de.tudarmstadt.ukp.inception.io.tei.dkprobackport.TeiConstants.TAG_TITLE; import static de.tudarmstadt.ukp.inception.io.tei.dkprobackport.TeiConstants.TAG_U; import static de.tudarmstadt.ukp.inception.io.tei.dkprobackport.TeiConstants.TAG_WORD; +import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.enableNamespaceSupport; +import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.newSaxParser; +import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.newSaxParserFactory; import static java.util.Arrays.asList; import static org.apache.commons.io.IOUtils.closeQuietly; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -48,6 +51,8 @@ import java.util.Stack; import java.util.zip.GZIPInputStream; +import javax.xml.parsers.ParserConfigurationException; + import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.uima.UimaContext; @@ -349,7 +354,8 @@ private void nextTeiElement() throws CollectionException, IOException source.setPublicId(currentResource.getLocation()); source.setSystemId(currentResource.getLocation()); - SAXReader reader = new SAXReader(); + var parser = newSaxParser(enableNamespaceSupport(newSaxParserFactory())); + var reader = new SAXReader(parser.getXMLReader()); Document xml = reader.read(source); final XPath teiPath = new Dom4jXPath("//tei:TEI"); @@ -363,7 +369,7 @@ private void nextTeiElement() throws CollectionException, IOException teiElementIterator = teiElements.iterator(); currentTeiElementNumber = 0; } - catch (DocumentException e) { + catch (DocumentException | ParserConfigurationException | SAXException e) { throw new IOException(e); } catch (JaxenException e) { diff --git a/inception/inception-io-tei/src/test/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReaderTest.java b/inception/inception-io-tei/src/test/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReaderTest.java new file mode 100644 index 00000000000..395a2b3ebd6 --- /dev/null +++ b/inception/inception-io-tei/src/test/java/de/tudarmstadt/ukp/inception/io/tei/dkprobackport/TeiReaderTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.io.tei.dkprobackport; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.uima.fit.factory.JCasFactory; +import org.dkpro.core.api.io.ResourceCollectionReaderBase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TeiReaderTest +{ + private static final String ENTITY_CONTENT = "ENTITY-CONTENT"; + + private static final String BODY = """ + + t +

p

s

+
+

MARKER_A %s MARKER_B

+
+ """; + + private @TempDir Path tempDir; + + @Test + void thatPlainTeiIsRead() throws Exception + { + var file = tempDir.resolve("doc.xml"); + Files.writeString(file, BODY.formatted("hello"), UTF_8); + + var text = read(file); + + assertThat(text).contains("MARKER_A hello MARKER_B"); + } + + @Test + void thatDoctypeIsRejected() throws Exception + { + var entityFile = tempDir.resolve("entity.txt"); + Files.writeString(entityFile, ENTITY_CONTENT, UTF_8); + + var file = tempDir.resolve("doc.xml"); + Files.writeString(file, """ + + ]> + """.formatted(entityFile.toUri()) + BODY.formatted("&ent;"), UTF_8); + + assertThatThrownBy(() -> read(file)) // + .satisfies(e -> assertThat(e).hasStackTraceContaining("DOCTYPE")) // + .satisfies(e -> assertThat(e).hasNoSuppressedExceptions()) // + .rootCause().message().doesNotContain(ENTITY_CONTENT); + } + + private String read(Path aFile) throws Exception + { + var reader = createReader(TeiReader.class, // + ResourceCollectionReaderBase.PARAM_SOURCE_LOCATION, aFile.getParent().toString(), // + ResourceCollectionReaderBase.PARAM_PATTERNS, "[+]" + aFile.getFileName()); + try { + var jcas = JCasFactory.createJCas(); + reader.hasNext(); + reader.getNext(jcas.getCas()); + return jcas.getDocumentText(); + } + finally { + reader.destroy(); + } + } +}