Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions inception/inception-io-perseus/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@
<artifactId>inception-model</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>de.tudarmstadt.ukp.inception.app</groupId>
<artifactId>inception-support-xml</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.springframework</groupId>
Expand All @@ -61,5 +66,9 @@
<groupId>org.dkpro.core</groupId>
<artifactId>dkpro-core-io-perseus-asl</artifactId>
</dependency>
<dependency>
<groupId>org.dkpro.core</groupId>
<artifactId>dkpro-core-api-resources-asl</artifactId>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,28 @@
*/
package de.tudarmstadt.ukp.inception.io.perseus;

import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.newSaxParser;
import static org.apache.uima.fit.factory.CollectionReaderFactory.createReaderDescription;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.uima.cas.CAS;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.collection.CollectionReaderDescription;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.dkpro.core.api.resources.CompressionUtils;
import org.dkpro.core.io.perseus.PerseusReader;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

import de.tudarmstadt.ukp.clarin.webanno.api.format.UimaReaderWriterFormatSupport_ImplBase;
import de.tudarmstadt.ukp.clarin.webanno.model.Project;
import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
import de.tudarmstadt.ukp.inception.io.perseus.config.PerseusFormatAutoConfiguration;

/**
Expand Down Expand Up @@ -71,4 +84,25 @@ public CollectionReaderDescription getReaderDescription(Project aProject,
{
return createReaderDescription(PerseusReader.class, aTSD);
}

@Override
public void read(SourceDocument aDocument, CAS aCas, File aFile)
throws ResourceInitializationException, IOException, CollectionException
{
validateXml(aFile);

super.read(aDocument, aCas, aFile);
}

static void validateXml(File aFile) throws IOException
{
try (var is = CompressionUtils.getInputStream(aFile.getName(),
Files.newInputStream(aFile.toPath()))) {
newSaxParser().parse(is, new DefaultHandler());
}
catch (ParserConfigurationException | SAXException e) {
throw new IOException("Unable to parse [" + aFile.getName()
+ "] - note that DOCTYPE declarations are not supported", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Licensed to the Technische Universität Darmstadt under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The Technische Universität Darmstadt
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package de.tudarmstadt.ukp.inception.io.perseus;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

import org.apache.uima.fit.factory.CasFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;

class PerseusFormatSupportTest
{
private static final String BODY = """
<treebank version="1.5" xml:lang="grc" format="aldt">
<sentence id="1" document_id="d" subdoc="s">
<word id="1" form="A%sB" lemma="x" postag="n-s---mn-" relation="SBJ" head="0"/>
</sentence>
</treebank>
""";

private @TempDir Path tempDir;

@Test
void thatPlainFileIsRead() throws Exception
{
var file = tempDir.resolve("doc.xml");
Files.writeString(file, BODY.formatted("-hello-"), UTF_8);

var cas = CasFactory.createCas();
new PerseusFormatSupport().read(new SourceDocument(), cas, file.toFile());

assertThat(cas.getDocumentText()).contains("A-hello-B");
}

@Test
void thatDoctypeIsRejected() throws Exception
{
var dtdFile = tempDir.resolve("doc.dtd");
Files.writeString(dtdFile, "<!ENTITY ent \"value\">", UTF_8);

var file = tempDir.resolve("doc.xml");
Files.writeString(file, """
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE treebank [ <!ENTITY % p SYSTEM "%s"> %p; ]>
""".replace("%s", dtdFile.toUri().toString()) + BODY.formatted("&ent;"), UTF_8);

var cas = CasFactory.createCas();
assertThatThrownBy(
() -> new PerseusFormatSupport().read(new SourceDocument(), cas, file.toFile())) //
.isInstanceOf(IOException.class) //
.hasMessageContaining("DOCTYPE");
assertThat(cas.getDocumentText()).isNull();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
import static de.tudarmstadt.ukp.inception.io.tei.dkprobackport.TeiConstants.TAG_TITLE;
import static de.tudarmstadt.ukp.inception.io.tei.dkprobackport.TeiConstants.TAG_U;
import static de.tudarmstadt.ukp.inception.io.tei.dkprobackport.TeiConstants.TAG_WORD;
import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.enableNamespaceSupport;
import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.newSaxParser;
import static de.tudarmstadt.ukp.inception.support.xml.XmlParserUtils.newSaxParserFactory;
import static java.util.Arrays.asList;
import static org.apache.commons.io.IOUtils.closeQuietly;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
Expand All @@ -48,6 +51,8 @@
import java.util.Stack;
import java.util.zip.GZIPInputStream;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.uima.UimaContext;
Expand Down Expand Up @@ -349,7 +354,8 @@ private void nextTeiElement() throws CollectionException, IOException
source.setPublicId(currentResource.getLocation());
source.setSystemId(currentResource.getLocation());

SAXReader reader = new SAXReader();
var parser = newSaxParser(enableNamespaceSupport(newSaxParserFactory()));
var reader = new SAXReader(parser.getXMLReader());
Document xml = reader.read(source);

final XPath teiPath = new Dom4jXPath("//tei:TEI");
Expand All @@ -363,7 +369,7 @@ private void nextTeiElement() throws CollectionException, IOException
teiElementIterator = teiElements.iterator();
currentTeiElementNumber = 0;
}
catch (DocumentException e) {
catch (DocumentException | ParserConfigurationException | SAXException e) {
throw new IOException(e);
}
catch (JaxenException e) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Licensed to the Technische Universität Darmstadt under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The Technische Universität Darmstadt
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package de.tudarmstadt.ukp.inception.io.tei.dkprobackport;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.nio.file.Files;
import java.nio.file.Path;

import org.apache.uima.fit.factory.JCasFactory;
import org.dkpro.core.api.io.ResourceCollectionReaderBase;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

class TeiReaderTest
{
private static final String ENTITY_CONTENT = "ENTITY-CONTENT";

private static final String BODY = """
<TEI xmlns="http://www.tei-c.org/ns/1.0">
<teiHeader><fileDesc><titleStmt><title>t</title></titleStmt>
<publicationStmt><p>p</p></publicationStmt><sourceDesc><p>s</p></sourceDesc>
</fileDesc></teiHeader>
<text><body><p>MARKER_A %s MARKER_B</p></body></text>
</TEI>
""";

private @TempDir Path tempDir;

@Test
void thatPlainTeiIsRead() throws Exception
{
var file = tempDir.resolve("doc.xml");
Files.writeString(file, BODY.formatted("hello"), UTF_8);

var text = read(file);

assertThat(text).contains("MARKER_A hello MARKER_B");
}

@Test
void thatDoctypeIsRejected() throws Exception
{
var entityFile = tempDir.resolve("entity.txt");
Files.writeString(entityFile, ENTITY_CONTENT, UTF_8);

var file = tempDir.resolve("doc.xml");
Files.writeString(file, """
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE TEI [ <!ENTITY ent SYSTEM "%s"> ]>
""".formatted(entityFile.toUri()) + BODY.formatted("&ent;"), UTF_8);

assertThatThrownBy(() -> read(file)) //
.satisfies(e -> assertThat(e).hasStackTraceContaining("DOCTYPE")) //
.satisfies(e -> assertThat(e).hasNoSuppressedExceptions()) //
.rootCause().message().doesNotContain(ENTITY_CONTENT);
}

private String read(Path aFile) throws Exception
{
var reader = createReader(TeiReader.class, //
ResourceCollectionReaderBase.PARAM_SOURCE_LOCATION, aFile.getParent().toString(), //
ResourceCollectionReaderBase.PARAM_PATTERNS, "[+]" + aFile.getFileName());
try {
var jcas = JCasFactory.createJCas();
reader.hasNext();
reader.getNext(jcas.getCas());
return jcas.getDocumentText();
}
finally {
reader.destroy();
}
}
}
Loading