diff --git a/codemeta.json b/codemeta.json index dd795999..ba6c5e87 100644 --- a/codemeta.json +++ b/codemeta.json @@ -93,7 +93,7 @@ } ], "softwareRequirements": "https://raw.githubusercontent.com/KnowledgeCaptureAndDiscovery/somef/refs/heads/master/pyproject.toml", - "version": "0.11.1", + "version": "0.11.2", "developmentStatus": "active", "issueTracker": "https://github.com/KnowledgeCaptureAndDiscovery/somef/issues", "@type": "SoftwareSourceCode" diff --git a/pyproject.toml b/pyproject.toml index 9545df5a..2fb1e2d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "somef" -version = "0.11.1" +version = "0.11.2" description = "SOftware Metadata Extraction Framework: A tool for automatically extracting relevant software metadata from a source code repository (README, package files, etc)." authors = ["Daniel Garijo "] readme = "README.md" diff --git a/src/somef/__init__.py b/src/somef/__init__.py index 08b269d3..eb0be577 100644 --- a/src/somef/__init__.py +++ b/src/somef/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = "0.11.1" +__version__ = "0.11.2" diff --git a/src/somef/export/json_export.py b/src/somef/export/json_export.py index cd001c03..2824afbf 100644 --- a/src/somef/export/json_export.py +++ b/src/somef/export/json_export.py @@ -443,7 +443,16 @@ def format_date(date_string): is_bibtex = False if constants.PROP_FORMAT in cit[constants.PROP_RESULT] and cit[constants.PROP_RESULT][constants.PROP_FORMAT] == "cff": - yaml_content = yaml.safe_load(cit[constants.PROP_RESULT]["value"]) + # yaml_content = yaml.safe_load(cit[constants.PROP_RESULT]["value"]) + cff_value = cit[constants.PROP_RESULT]["value"] + try: + yaml_content = yaml.safe_load(cff_value) + except Exception: + # Remove HTML tags that can break YAML parsing + # ej:" " has unescaped quotes + cleaned = re.sub(constants.REGEXP_CLEAN_HTML_TAGS, '', cff_value) + yaml_content = yaml.safe_load(cleaned) + preferred_citation = yaml_content.get("preferred-citation", {}) doi = yaml_content.get("doi") or preferred_citation.get("doi") identifiers = yaml_content.get("identifiers", []) @@ -514,6 +523,8 @@ def format_date(date_string): "@type": "Organization", "name": name } + else: + continue if family_name and given_name and orcid: key = (family_name.lower(), given_name.lower()) diff --git a/src/somef/process_files.py b/src/somef/process_files.py index ae3daa6e..91fbfd17 100644 --- a/src/somef/process_files.py +++ b/src/somef/process_files.py @@ -611,7 +611,11 @@ def get_file_content_or_link(repo_type, file_path, owner, repo_name, repo_defaul try: yaml_content = yaml.safe_load(file_text) except Exception: - yaml_content = None + cleaned = re.sub(constants.REGEXP_CLEAN_HTML_TAGS, '', file_text) + try: + yaml_content = yaml.safe_load(cleaned) + except Exception: + yaml_content = None if yaml_content: license_value = yaml_content.get("license") diff --git a/src/somef/test/test_JSON_export.py b/src/somef/test/test_JSON_export.py index 924a0f64..4917bc99 100644 --- a/src/somef/test/test_JSON_export.py +++ b/src/somef/test/test_JSON_export.py @@ -1144,4 +1144,34 @@ def test_issue_533_choosealicense_badge(self): os.remove(output_path) - + def test_issue_1050(self): + """citation.cff yaml parsing with unescaped quotes in abstract should not crash""" + somef_cli.run_cli(threshold=0.8, + ignore_classifiers=False, + repo_url=None, + local_repo=test_data_repositories + "nemo", + output=test_data_path + "test_issue_1050.json", + pretty=True, + readme_only=False) + + + with open(test_data_path + "test_issue_1050.json") as f: + json_content = json.load(f) + + cff_entries = [ + c for c in json_content.get(constants.CAT_CITATION, []) + if "CITATION.cff" in c.get("source", "") + ] + assert len(cff_entries) > 0, "CITATION.cff not found in citations" + + cff_entry = json_content[constants.CAT_CITATION][0]["result"] + assert cff_entry.get("title") == "NEMO: A Stellar Dynamics Toolbox" + assert cff_entry.get("type") == "SoftwareApplication" + + cff_authors = cff_entry.get("author", []) + assert len(cff_authors) == 3 + orcid_urls = [a.get("url") for a in cff_authors] + assert all(u and u.startswith("https://orcid.org/") for u in orcid_urls) + + os.remove(test_data_path + "test_issue_1050.json") + diff --git a/src/somef/test/test_codemeta_export.py b/src/somef/test/test_codemeta_export.py index 7742cbe6..0a0dc07e 100644 --- a/src/somef/test/test_codemeta_export.py +++ b/src/somef/test/test_codemeta_export.py @@ -968,6 +968,29 @@ def test_issue_1025_orcid(self): os.remove(output_path) + def test_codemeta_nemo_cff_issue_1050(self): + """CITATION.cff with HTML in abstract exports with unescaped quotes correctly to codemeta""" + output_path = test_data_path + 'test_codemeta_nemo.json' + somef_cli.run_cli(threshold=0.8, + ignore_classifiers=False, + repo_url=None, + local_repo=test_data_repositories + "nemo", + output=None, + codemeta_out=output_path, + pretty=True, + readme_only=False) + + with open(output_path) as f: + json_content = json.load(f) + + credit_text = json_content.get(constants.CAT_CODEMETA_CREDITTEXT, []) + assert len(credit_text) > 0 + assert any("NEMO" in ct.upper() for ct in credit_text) + assert any("BARNES" in ct.upper() for ct in credit_text) + + os.remove(output_path) + + @classmethod def tearDownClass(cls): """delete temp file JSON just if all the test pass""" diff --git a/src/somef/test/test_data/repositories/nemo/CITATION.cff b/src/somef/test/test_data/repositories/nemo/CITATION.cff new file mode 100644 index 00000000..812fbe65 --- /dev/null +++ b/src/somef/test/test_data/repositories/nemo/CITATION.cff @@ -0,0 +1,23 @@ +cff-version: 1.2.0 +message: "Please cite the following works when using this software: https://ui.adsabs.harvard.edu/abs/1995ASPC...77..398T" +authors: +- family-names: Barnes + given-names: Joshua + orcid: https://orcid.org/0000-0002-3371-1082 +- family-names: Hut + given-names: Piet + orcid: https://orcid.org/0000-0001-7565-5843 +- family-names: Teuben + given-names: Peter + orcid: https://orcid.org/0000-0003-1774-3436 +title: "NEMO: A Stellar Dynamics Toolbox" +version: 4.4.0 +date-released: 2022-11-01 +identifiers: + - type: "ascl-id" + value: "1010.051" + - type: "doi" + value: PLACEHOLDER + - type: "bibcode" + value: "2010ascl.soft10051B" +abstract: "NEMO is an extendible Stellar Dynamics Toolbox, following an Open-Source Software model. It has various programs to create, integrate, analyze and visualize N-body and SPH like systems, following the pipe and filter architecture. In addition there are various tools to operate on images, tables and orbits, including FITS files to export/import to/from other astronomical data reduction packages. A large growing fraction of NEMO has been contributed by a growing list of authors. The source code consist of a little over 4000 files and a little under 1,000,000 lines of code and documentation, mostly C, and some C++ and Fortran. NEMO development started in 1986 in Princeton (USA) by Barnes, Hut and Teuben. See also ZENO (ascl:1102.027) for the version that Barnes maintains." diff --git a/src/somef/test/test_data/repositories/nemo/README.md b/src/somef/test/test_data/repositories/nemo/README.md new file mode 100644 index 00000000..e8dbaf97 --- /dev/null +++ b/src/somef/test/test_data/repositories/nemo/README.md @@ -0,0 +1,185 @@ +![A Real Bar](docs/figures/realbar1.png) + +NEMO is a toolbox for stellar dynamics, particle simulations, stellar orbits, +image processing and tabular data manipulation. Documentation is maintained +in the github pages, https://teuben.github.io/nemo , and a manual in +https://astronemo.readthedocs.io + +Unless otherwise specified, all code has been written by a human. + + +## History + +This is the 4th major release of NEMO, and although data are compatible +with earlier releases, old source code may need to be tweaked a +bit to compile and link in the newer releases. Some compatibility with ZENO +is also advertised. A brief history of NEMO: + + * NEMO V1: IAS release (Barnes, Hut & Teuben, 1987) + * NEMO V2: UMD release (Teuben, 1994) + * NEMO V3: UMD release (Teuben, 2001) in CVS, w/ autoconf, manybody, starlab and partiview + * NEMO V4: UMD/ESO release (2017) now maintained in github + +A related package, ZENO, was spun off NEMO V1, and is maintained by Josh Barnes. +Other packages that geneologically came after NEMO are StarLab, ACS and AMUSE +(see also https://ascl.net for code references): + + * NEMO: ascl:1010.051 + * ZENO: ascl:1102.027 (normally installed in $NEMO/usr/zeno) + * STARLAB: ascl:1010.076 (optionally installed in $NEMO/local/starlab) + * ACS: [https://artcompsci.org](https://web.archive.org/web/20240206050455/http://www.artcompsci.org) + * AMUSE: ascl:1107.007 + +## Optional Packages + + + PGPLOT: ascl:1103.002 + CFITSIO: ascl:1010.001 + WCSLIB: ascl:1108.003 + glnemo2: ascl:1110.008 + gyrfalcON: ascl:1402.031 (included with NEMO) + HDF4 + HDF5 https://www.hdfgroup.org + netcdf4 + gsl + plplot + unsio + uns_project + wcstools + +Tools you will need to have pre-installed: the C/C++/Fortran +compilers, (t)csh, and git. For graphics it's probably useful to have +pgplot, but the default ps driver works fine just to get started. +We are looking for more portable full graphics, as an alternative +to pgplot. + +The files in $NEMO/src/scripts/linux describe the actual package +names for different linux distros that should lead to success. + + +## Installation + +There are a few different ways to install NEMO. +We cover an annotated example in this +[bash notebook](scripts/notebooks/installing.ipynb) +or the deprecated +[bash notebook](example.ipynb). + +And here is a simple +example that works most of the time on most Linux (including WSL) distros: + + git clone https://github.com/teuben/nemo + cd nemo + ./configure --with-yapp=pgplot + make build check bench5 + source nemo_start.sh + +If you plan to modify code and submit pull requests, the github CLI is recommended, +though you can of course also clone the upstream manually +(see also [CONTRIBUTING.md](CONTRIBUTING.md)): + + gh repo fork https://github.com/teuben/nemo + +After installation, rebuilding NEMO to ensure you have all updates can be done as follows: + + cd $NEMO + git pull + make rebuild + +this may be overkill for some situations, but gets the job done. + +### python + +There is now a small python component to NEMO, in the **nemopy** module. + + cd $NEMO + pip install -e . + +but this will depend on the details of how your python environment exists (virtual, conda etc.). +We leave this to the user. + +## Examples + +If you want to quickly see something work, here are the commands to +make a classic 1911 Plummer sphere of just 10 particles, print the positions, plot +the positions and view the contents of its binary file: + + mkplummer p10.dat 10 + snapprint p10.dat + snapplot p10.dat + tsf p10.dat + +and here is an example of creating the 4 major data objects on the fly in NEMO +(table, snapshot, image, orbit), showing off the command line interface and +use of Unix pipes, with a dash denoting the piped file: + + tabgen - | tabplot - + mkplummer - 100 | snapplot - + ccdgen out=- object=gauss spar=1,20 size=128 | ccdplot - + mkorbit - 0 1 0 0.4 0 0 potname=plummer | orbint - - nsteps=1000 dt=0.05 | orbplot - + +or graphically showing the pipes through arrows: + +```mermaid + graph LR; + A[tabgen]-->B[tabplot] + + C[mkplummer]-->D[snapplot] + + E[ccdgen]-->F[ccdplot] + + G[mkorbit]-->H[orbint] + H-->I[orbplot] +``` + +There are more examples of scripts and figures in +https://teuben.github.io/nemo/examples/ +and an example ipython notebook is shown here +https://github.com/teuben/nemo/blob/master/nemo_start_example.ipynb +for something completely different. + +A reproducable example is given in this notebook. To run this from the command line +and print the phase space coordinates of the first particle, use this: + + mkplummer - 10 seed=123| snapprint - | head -1 + ### nemo Debug Info: x y z vx vy vz + -0.609486 -0.221687 -0.450963 -0.0766784 -0.209397 0.396561 + + +## Documentation and Help + +There are several additional entry points if you are starting out with NEMO, +between them there is unavoidable duplication, but here they are: + +* readthedocs: https://astronemo.readthedocs.io/en/latest/ +* github pages: https://teuben.github.io/nemo (yes, these are the 1990s style webpages we used to maintain) +* all unix style man pages for all levels: https://teuben.github.io/nemo/man_html/index1.html +* the nemo man page https://teuben.github.io/nemo/man_html/nemo.1.html +* software carpentry: https://teuben.github.io/nemo-lesson +* AAS 443 [iPoster](https://aas243-aas.ipostersessions.com/?s=85-F6-32-6F-83-00-4E-79-54-7F-C0-25-77-D7-0D-7B) +* an installation example in bash jupyter notebook style: https://github.com/teuben/nemo/blob/master/scripts/notebooks/installing.ipynb +* contributing to NEMO: https://github.com/teuben/nemo/blob/master/CONTRIBUTING.md + +## Citation + +Please use the following citation if you use NEMO in your work + + + @INPROCEEDINGS{1995ASPC...77..398T, + author = {{Teuben}, P.}, + title = "{The Stellar Dynamics Toolbox NEMO}", + booktitle = {Astronomical Data Analysis Software and Systems IV}, + year = 1995, + editor = {{Shaw}, R.~A. and {Payne}, H.~E. and {Hayes}, J.~J.~E.}, + series = {Astronomical Society of the Pacific Conference Series}, + volume = {77}, + month = jan, + pages = {398}, + adsurl = {https://ui.adsabs.harvard.edu/abs/1995ASPC...77..398T}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} + } + + +![A Real Bar](docs/figures/realbar1.png) + +[taken from this example script](https://teuben.github.io/nemo/examples/realbar.html) \ No newline at end of file diff --git a/src/somef/utils/constants.py b/src/somef/utils/constants.py index 12e19eab..3b0a035a 100644 --- a/src/somef/utils/constants.py +++ b/src/somef/utils/constants.py @@ -106,6 +106,8 @@ # Detect copyright information in license files. REGEXP_COPYRIGHT = r"copyright\s*(?:\(c\)|©|\(C\))?\s*\{?(\d{4}(?:-\d{4})?)\}?\s*\{?([^\n}]+)\}?" +REGEXP_CLEAN_HTML_TAGS = r'<[^>]+>' + LICENSES_DICT = { "Apache License 2.0": {"regex": REGEXP_APACHE, "spdx_id": "Apache-2.0"}, "GNU General Public License v3.0": {"regex": REGEXP_GPL3, "spdx_id": "GPL-3.0"},