This tutorial covers Python tools related to GATE:
Aims:
conda create -n gatenlp python=3.9
conda activate gatenlp
To install most recent release and install all dependencies
(without [all]
only minimum dependencies are installed!):
pip install -U gatenlp[all]
Also install support for jupyter notebookd and for showing the slides:
pip install jupyter notebook ipython ipykernel RISE
Create kernel for the conda environment:
python -m ipykernel install --user --name gatenlp --display-name "Python-gatenlp"
GATE_HOME
)jupyter notebook module11-python.ipynb
jupyter notebook
, choose New -> Python-gatenlp ipython
and enter python codeIf kernel error in Jupyter, try something like (Anaconda bug, apparently):
python C:\Users\USERNAME\miniconda3\envs\gatenlp\Scripts\pywin32_postinstall.py -install
from gatenlp import Document
Create a document from some text/string:
doc1 = Document("This is a small test document")
Print the document:
print(doc1)
Document(This is a small test document,features=Features({}),anns=[])
In a notebook, documents are visualized using the html-viewer when a document is the last value of a cell or when display(doc1)
or when document.show()
is used:
# from IPython.display import display
doc1
doc1.show()
Document.load(some_location, ...)
fmt
parametersome_location
can be file or URLdoc2 = Document.load('./data/document-testing.txt')
doc2
thedocument.save(location, ...)
fmt
parameterbdocjs
(JSON, default), bdocym
(YAML, slow), bdocmp
(MessagePack, compact)doc1.save("myfirstdocument.bdocjs")
with open("myfirstdocument.bdocjs", "rt", encoding="utf-8") as infp:
print(infp.read())
{"annotation_sets": {}, "text": "This is a small test document", "features": {}, "offset_type": "p", "name": ""}
doc1.save("myfirstdocument.bdocym") # use YAML serialization
with open("myfirstdocument.bdocym", "rt", encoding="utf-8") as infp:
print(infp.read())
annotation_sets: {} features: {} name: '' offset_type: p text: This is a small test document
# Can also "save" to memory/string, here the format is needed!
doc1.save_mem(fmt="bdocjs")
'{"annotation_sets": {}, "text": "This is a small test document", "features": {}, "offset_type": "p", "name": ""}'
import datetime
doc1.features["loading_date"] = str(datetime.datetime.now())
doc1.features["purpose"] = "Testing gatenlp."
doc1.features["numeric_value"] = 22
doc1.features["dict_of_objects"] = {"dict_key": "dict_value", "a_list": [1,2,3,4,5]}
doc1.features["_tmp1"] = "some value"
doc1.features["__tmp2"] = 12345
doc1
print("1:", doc1.features["purpose"])
print("2:", doc1.features.get("doesntexist"))
print("3:", doc1.features.get("doesntexist", "NA!"))
1: Testing gatenlp. 2: None 3: NA!
for name, value in doc1.features.items():
print(f"{name}: {value}")
loading_date: 2021-02-27 10:36:27.181482 purpose: Testing gatenlp. numeric_value: 22 dict_of_objects: {'dict_key': 'dict_value', 'a_list': [1, 2, 3, 4, 5]} _tmp1: some value __tmp2: 12345
Lets check how the document with features is serialized to "bdocjs" (JSON) format:
import pprint, json
js_str = doc1.save_mem(fmt="bdocjs")
js = json.loads(js_str)
pprint.pprint(js)
{'annotation_sets': {}, 'features': {'_tmp1': 'some value', 'dict_of_objects': {'a_list': [1, 2, 3, 4, 5], 'dict_key': 'dict_value'}, 'loading_date': '2021-02-27 10:36:27.181482', 'numeric_value': 22, 'purpose': 'Testing gatenlp.'}, 'name': '', 'offset_type': 'p', 'text': 'This is a small test document'}
add
method of the set# create and get an annotation set with the name "Set1"
annset = doc1.annset("Set1")
#Now, add an annotation, this method returns the newly created annotation
annset.add(0,4,"AnnType1")
Annotation(0,4,AnnType1,features=Features({}),id=0)
annset.add(0, 4, "Token", {"id": "token1'"})
annset.add(5, 7, "Token", {"id": "token2'"})
annset.add(8, 9, "Token", {"id": "token3'"})
annset.add(10, 15, "Token", {"id": "token4'"})
annset.add(16, 20, "Token", {"id": "token5"})
annset.add(21, 29, "Token", {"id": "token6"})
annset.add(0, 29, "Sentence", {"what": "The first 'sentence' annotation"});
for ann in annset:
print(ann)
Annotation(0,4,AnnType1,features=Features({}),id=0) Annotation(0,4,Token,features=Features({'id': "token1'"}),id=1) Annotation(0,29,Sentence,features=Features({'what': "The first 'sentence' annotation"}),id=7) Annotation(5,7,Token,features=Features({'id': "token2'"}),id=2) Annotation(8,9,Token,features=Features({'id': "token3'"}),id=3) Annotation(10,15,Token,features=Features({'id': "token4'"}),id=4) Annotation(16,20,Token,features=Features({'id': 'token5'}),id=5) Annotation(21,29,Token,features=Features({'id': 'token6'}),id=6)
doc1
ann0 = annset.get(0) # get by annotation id
print("Annotation id=0:", ann0)
annset.remove(ann0) # remove the annotation with the annotation id of ann1
ann1 = annset.get(1)
print("Annotation id=1:", ann1)
annset.remove(1) # remove the annotation with the given id
annset.remove([2,3,4]) # remove a whole list of annotations
print("After some anns removed ", annset)
annset.clear()
print("After set cleared: ", annset)
doc1.remove_annset("Set1")
Annotation id=0: Annotation(0,4,AnnType1,features=Features({}),id=0) Annotation id=1: Annotation(0,4,Token,features=Features({'id': "token1'"}),id=1) After some anns removed AnnotationSet([Annotation(0,29,Sentence,features=Features({'what': "The first 'sentence' annotation"}),id=7), Annotation(16,20,Token,features=Features({'id': 'token5'}),id=5), Annotation(21,29,Token,features=Features({'id': 'token6'}),id=6)]) After set cleared: AnnotationSet([])
Let's load and view an example document to demonstrate this:
doc3 = Document.load("data/ann-relations.bdocjs")
doc3.show(htmlid="view1")
# make a variable for each annotation type
for anntype in list(doc3.annset("set1").type_names):
vars()[anntype.lower()] = doc3.annset("set1").with_type(anntype).for_idx(0)
print("Ann2 isoverlapping Ann1:", ann2.isoverlapping(ann1))
print("Ann2 isbefore Ann3:", ann2.isbefore(ann3))
print("Ann3 isafter Ann2:", ann3.isafter(ann2))
print("Ann1 iscovering Ann5:", ann1.iscovering(ann5))
print("Ann3 iscoextensive Ann9:", ann3.iscoextensive(ann9))
print("Ann6 iswithin Ann1:", ann6.iswithin(ann1))
print("Ann4 isrightoverlapping Ann1:", ann4.isrightoverlapping(ann1))
Ann2 isoverlapping Ann1: True Ann2 isbefore Ann3: True Ann3 isafter Ann2: True Ann1 iscovering Ann5: True Ann3 iscoextensive Ann9: True Ann6 iswithin Ann1: True Ann4 isrightoverlapping Ann1: True
from gatenlp import Span
span1 = Span(3,4)
span2 = ann2.span
span3 = doc3.annset("set1").span
span4 = Span(ann5)
print([f"span{i}: {s}" for i, s in enumerate([span1, span2, span3, span4])])
['span0: Span(3,4)', 'span1: Span(0,6)', 'span2: Span(0,45)', 'span3: Span(12,18)']
set1 = doc3.annset("set1") # "attached" set
print("Within Ann1: ", [a.type for a in set1.within(ann1)])
print("Coextensive with Ann3:", [a.type for a in set1.coextensive(ann3)])
print("Coextensive with span of Ann3:", [a.type for a in set1.coextensive(ann3.span)])
Within Ann1: ['Ann10', 'Ann5', 'Ann3', 'Ann7', 'Ann9', 'Ann11', 'Ann6', 'Ann8', 'Ann12'] Coextensive with Ann3: ['Ann9'] Coextensive with span of Ann3: ['Ann3', 'Ann9']
print("Size of set1:", len(set1))
subset1 = set1.within(ann1)
print("Size of subset1:", len(subset1))
Size of set1: 12 Size of subset1: 9
# try to add an annotation to subset1:
try:
subset1.add(2,3,"ANewOne")
except Exception as ex:
print("Got exception:", ex)
Got exception: Cannot add an annotation to an immutable annotation set
# make the set mutable and try again
subset1.immutable = False
subset1.add(2,3,"ANewOne")
print("Size of set1:", len(set1))
print("Size of subset1:", len(subset1))
print("Is set1 detached:", set1.isdetached())
print("Is subset1 detached:", subset1.isdetached())
Size of set1: 12 Size of subset1: 10 Is set1 detached: False Is subset1 detached: True
subset1
, NOT the original setsubset1.clone_anns()
Supported formats:
# lets load and view the main GateNLP documentation page:
doc4 = Document.load("https://gatenlp.github.io/python-gatenlp/", fmt="html")
doc4
Use: doc.show(annsets=["set1", ("set2", "type1"), ("set3", ["type1", "type2"])]
doc4.show(annsets=[("Original markups", ["h1","h2","a","li"])])
doc4.save("gatenlp-doc.html", fmt="html-ann-viewer", notebook=False, stretch_height=True)
from IPython.display import IFrame
IFrame("gatenlp-doc.html", 900,400)
offset_type
is either p
or j
doc = corpus[2]
set: corpus[3] = doc
store(doc)
to save a document to the index stored in the document featureappend(doc)
to add a new document to the corpusNone
None
indicates absence of documentNone
indicates that document should get removed or should not get updatedstore
methodfrom gatenlp.corpora import ListCorpus
texts = ["this is text one", "here is text two", "and this is text three"]
docs = [Document(t) for t in texts]
lcorp = ListCorpus(docs)
doc1 = lcorp[1]
print(doc1.features)
lcorp.store(doc1)
Features({'__idx': 1, '__idx_140270484012728': 1})
from gatenlp.corpora import DirFilesCorpus
corp1 = DirFilesCorpus("data/dir1") # get all the matching filenames from the directory
print("Number of documents:", len(corp1))
doc1 = corp1[2] # actually read the document from the directory
print("Text for idx=2:", doc1.text)
print("Features for idx=2:", doc1.features)
doc1.annset().add(0,len(doc1.text), "Document", dict(what="test document"))
# this writes the document back to the file:
corp1.store(doc1)
# could also have used: corp1[2] = doc1
Number of documents: 4 Text for idx=2: This is another document for testing which mentions John Smith. Features for idx=2: Features({'__idx_140270484365552': 2, '__idx': 2, '__relpath': 'doc2.bdocjs', '__abspath': 'data/dir1/doc2.bdocjs'})
NumberedDirFilesCorpus
: create a directory tree where the path represents digits of a large number000/002/341.bdoc
for element number 2341 of 600000000 totalEveryNthCorpus
: wrap a corpus and access only elements $k*i + o$ for $i = 0..\lfloor(n/k)\rfloor$ShuffledCorpus
: random re-ordering of the elements in the wrapped corpusCachedCorpus
: store retrieved elements from a (slow) base corpus in a (fast) cache corpusappend(doc)
to add Document instancesclose()
to end writingwith documentdestination as dest:
patternBdocjsLinesFileSource/Destination
: one line of bdocjs serialization per documentTsvFileSource
: one column in a TSV file contains the text, other columns can be stored in featuresPandasDfSource
: similar to TSV source, but for a Pandas data framefrom gatenlp.corpora import TsvFileSource
tsvsrc1 = TsvFileSource("data/mytsvfile.tsv", text_col="text", feature_cols=dict(src="source",year="year"))
for doc in tsvsrc1:
print(doc)
Document(This is the text of the first row. It has several sentences.,features=Features({'src': 'source1', 'year': '2005'}),anns=[]) Document(Text of the second row.,features=Features({'src': 'source1', 'year': '2006'}),anns=[]) Document(Another text, this time of the third row. ,features=Features({'src': 'source2', 'year': '2001'}),anns=[]) Document(And here another, from the fourth row.,features=Features({'src': 'source3', 'year': '2013'}),anns=[])
from gatenlp.corpora import PandasDfSource
try: # this requires Pandas!
import pandas as pd, csv
df = pd.read_csv("data/mytsvfile.tsv", sep="\t", quotechar=None, index_col=None, quoting=csv.QUOTE_NONE)
pdsrc1 = PandasDfSource(df, text_col="text", data_cols=["source", "year"])
for doc in pdsrc1:
print(doc)
except:
print("Pandas not installed")
Document(This is the text of the first row. It has several sentences.,features=Features({'__data': {'source': 'source1', 'year': 2005}}),anns=[]) Document(Text of the second row.,features=Features({'__data': {'source': 'source1', 'year': 2006}}),anns=[]) Document(Another text, this time of the third row. ,features=Features({'__data': {'source': 'source2', 'year': 2001}}),anns=[]) Document(And here another, from the fourth row.,features=Features({'__data': {'source': 'source3', 'year': 2013}}),anns=[])
ar-ud-train.conllu
from gatenlp.corpora.conll import ConllUFileSource
src = ConllUFileSource("data/ar-tiny.conllu", group_by="doc", n=1)
corp = list(src)
print(len(corp))
3
corp[0].show(doc_style="direction: rtl; font-size: 1.5em; line-height: 1.5;")
Annotator
from gatenlp.corpora import ListCorpus
from gatenlp.processing.pipeline import Pipeline
from gatenlp.processing.annotator import AnnotatorFunction
from gatenlp.processing.executor import SerialCorpusExecutor
texts = ["Some text.", "Another text.", "Also some text here.", "And this is also some text."]
docs = [Document(t) for t in texts]
corp = ListCorpus(docs)
def annfunc1(doc):
doc.annset().add(0,3,"Ann1")
return doc
def annfunc2(doc):
doc.annset("set1").add(1,4,"Type1")
return doc
ann1 = AnnotatorFunction(annfunc1)
ann2 = AnnotatorFunction(annfunc2)
pipeline = Pipeline()
pipeline.add(ann1, name="FirstAnnotator")
pipeline.add(ann2, name="SecondAnnotator")
exe = SerialCorpusExecutor(pipeline, corpus=corp)
exe()
corp[2]
# use corp as source and create another ListCorpus as destination
corpnew = ListCorpus([])
exe2 = SerialCorpusExecutor(pipeline, source=corp, destination=corpnew)
exe2()
print("Length of corpnew:", len(corpnew))
print(f"in={exe2.n_in}, out={exe2.n_out}, none={exe2.n_none}, ok={exe2.n_ok}, err={exe2.n_err}")
corpnew[2]
Length of corpnew: 4 in=4, out=4, none=0, ok=4, err=0
Preparation:
pip install -U spacy
(not necessary if gatenlp[all]
was used)python -m spacy download en_core_web_sm
import spacy
print("Spacy version:", spacy.__version__)
from gatenlp.lib_spacy import AnnSpacy
nlp = spacy.load("en_core_web_sm")
annotator = AnnSpacy(pipeline=nlp, outsetname="Spacy")
doc2.annset("Spacy").clear() # avoid annotation duplication when running several times
doc2 = annotator(doc2)
Spacy version: 2.3.2
# Adapt size of viewer
from IPython.core.display import display, HTML
display(HTML("<style>#view2-wrapper { font-size: 80% !important; } #view2-row1 {height: 15em; min-height: 5em;}</style>"))
doc2.show(htmlid="view2")