Difference between revisions of "ClearlyDefinedObject"

Latest revision as of 16:19, 26 January 2021

Examples of Definitions

1. With source and gitSha

{
      "described": {
        "releaseDate": "2020-12-31",
        "urls": {
          "registry": "https://github.com/react-component/image",
          "version": "https://github.com/react-component/image/tree/0df9fa84ae6057e8f0cc2ddcbea0a409f9e78d69",
          "download": "https://github.com/react-component/image/archive/0df9fa84ae6057e8f0cc2ddcbea0a409f9e78d69.zip"
        },
        "hashes": {
          "gitSha": "0df9fa84ae6057e8f0cc2ddcbea0a409f9e78d69"
        },
        "files": 39,
        "tools": [
          "clearlydefined/1.3.0",
          "licensee/9.13.0",
          "scancode/3.2.2"
        ],
        "toolScore": {
          "total": 100,
          "date": 30,
          "source": 70
        },
        "sourceLocation": {
          "type": "git",
          "provider": "github",
          "namespace": "react-component",
          "name": "image",
          "revision": "0df9fa84ae6057e8f0cc2ddcbea0a409f9e78d69",
          "url": "https://github.com/react-component/image/tree/0df9fa84ae6057e8f0cc2ddcbea0a409f9e78d69"
        },
        "score": {
          "total": 100,
          "date": 30,
          "source": 70
        }
      },
      "licensed": {
        "declared": "MIT",
        "toolScore": {
          "total": 76,
          "declared": 30,
          "discovered": 1,
          "consistency": 15,
          "spdx": 15,
          "texts": 15
        },
        "facets": {
          "core": {
            "attribution": {
              "unknown": 38,
              "parties": [
                "Copyright (c) 2015-present Alipay.com, https://www.alipay.com"
              ]
            },
            "discovered": {
              "unknown": 37,
              "expressions": [
                "MIT"
              ]
            },
            "files": 39
          }
        },
        "score": {
          "total": 76,
          "declared": 30,
          "discovered": 1,
          "consistency": 15,
          "spdx": 15,
          "texts": 15
        }
      },
      "coordinates": {
        "type": "git",
        "provider": "github",
        "namespace": "react-component",
        "name": "image",
        "revision": "0df9fa84ae6057e8f0cc2ddcbea0a409f9e78d69"
      },
      "_meta": {
        "schemaVersion": "1.6.1",
        "updated": "2021-01-04T08:27:13.359Z"
      },
      "scores": {
        "effective": 88,
        "tool": 88
      }
    }

In the definitions where we get gitSha and Sourcelocation, we can map this kind of data with raw_extrensic_metadata by mapping gitSha with Core identifiers by adding "swh:1:cnt:" as prefix and url under SourceLocation as origin.

2. Definitions with Per file information

{
  "described": {
    "releaseDate": "2021-01-06",
    "sourceLocation": {
      "type": "git",
      "provider": "github",
      "namespace": "microsoft",
      "name": "rushstack",
      "revision": "1800f27e0506a200549004f8bb51dba1f8ae6d80",
      "url": "https://github.com/microsoft/rushstack/tree/1800f27e0506a200549004f8bb51dba1f8ae6d80"
    },
    "urls": {
      "registry": "https://npmjs.com/package/@rushstack/typings-generator",
      "version": "https://npmjs.com/package/@rushstack/typings-generator/v/0.3.0",
      "download": "https://registry.npmjs.com/@rushstack/typings-generator/-/typings-generator-0.3.0.tgz"
    },
    "hashes": {
      "sha1": "b0a78008deb61b446eec7624d078f1571d296404",
      "sha256": "373c0ad17af137f27bb89fa4e744522959b774d97b5a705af79b090228bff192"
    },
    "files": 19,
    "tools": [
      "clearlydefined/1.3.4",
      "licensee/9.13.0",
      "scancode/3.2.2"
    ],
    "toolScore": {
      "total": 100,
      "date": 30,
      "source": 70
    },
    "score": {
      "total": 100,
      "date": 30,
      "source": 70
    }
  },
  "licensed": {
    "declared": "MIT",
    "toolScore": {
      "total": 84,
      "declared": 30,
      "discovered": 9,
      "consistency": 15,
      "spdx": 15,
      "texts": 15
    },
    "facets": {
      "core": {
        "attribution": {
          "unknown": 12,
          "parties": [
            "Copyright (c) Microsoft Corporation."
          ]
        },
        "discovered": {
          "unknown": 11,
          "expressions": [
            "MIT"
          ]
        },
        "files": 19
      }
    },
    "score": {
      "total": 84,
      "declared": 30,
      "discovered": 9,
      "consistency": 15,
      "spdx": 15,
      "texts": 15
    }
  },
  "files": [
    {
      "path": "package/CHANGELOG.json",
      "hashes": {
        "sha1": "7bebb6bd48dd52f747d75e6eabc6471850a70f9a",
        "sha256": "647857d84ef3751afa46ae806c53d2f21d6f0b8f9600dc8a56aaf265ed728301"
      }
    },
    {
      "path": "package/LICENSE",
      "license": "MIT",
      "natures": [
        "license"
      ],
      "attributions": [
        "Copyright (c) Microsoft Corporation."
      ],
      "hashes": {
        "sha1": "da7010e6cf672f7852385c3e9beed970a294b3ac",
        "sha256": "45b02543066943768703985cc6c97976cf14b8696f45166141f1527fbb6b69c7"
      },
      "token": "45b02543066943768703985cc6c97976cf14b8696f45166141f1527fbb6b69c7"
    },
    {
      "path": "package/package.json",
      "license": "MIT",
      "hashes": {
        "sha1": "3551d742ea583574a4e3c255f3d3c96e4a523855",
        "sha256": "dc3d7454045ef7a9f3df9e5769b03d7593a111d7728861cab1028336289d8bd9"
      },
      "token": "dc3d7454045ef7a9f3df9e5769b03d7593a111d7728861cab1028336289d8bd9"
    },
    {
      "path": "package/lib/index.js",
      "license": "MIT",
      "attributions": [
        "Copyright (c) Microsoft Corporation."
      ],
      "hashes": {
        "sha1": "e2707f391baf9a6daebf06f8a4a8e72e9c7d6bc9",
        "sha256": "3326e3aa4c50d5dccbac386bd691012b9908ef3c391e124b9c542a442d3dafc1"
      }
    },
    {
      "path": "package/lib/index.js.map",
      "license": "MIT",
      "attributions": [
        "Copyright (c) Microsoft Corporation."
      ],
      "hashes": {
        "sha1": "4928f7046478eb90206a298d5b4fff31a44c665f",
        "sha256": "a9ccb69e65fa0c9b7a12c9a1d6fce5d4e2ef31d446bd0438ca01657b7d398595"
      }
    },
  ],
  "coordinates": {
    "type": "npm",
    "provider": "npmjs",
    "namespace": "@rushstack",
    "name": "typings-generator",
    "revision": "0.3.0"
  },
  "_meta": {
    "schemaVersion": "1.6.1",
    "updated": "2021-01-07T00:19:04.855Z"
  },
  "scores": {
    "effective": 92,
    "tool": 92
  }
}

In the definitions where we get sha1 of every file, we can map it with raw_extrensic_metadata by mapping sha1 with Core identifiers by finding respective sha1git under the content table, and then adding "swh:1:cnt:" as prefix

described

Contains releaseDate, urls, hashes, files, tools, toolscore :- (date and source) , sourceLocation (only those who have a source), score :- (date and source)

licensed

Declared license, tool score (spdx, consistency, texts, declared) binary and (discovered) computed. (https://github.com/clearlydefined/license-score/blob/master/ClearlyLicensedMetrics.md)

coordinates

Coordinates are used to identify various elements in ClearlyDefined. For example, components to harvest, tool outputs, source locations. The coordinates for an entity has at least five parts: type, provider, namespace, name and revision.

type – the form of the entity being identified. For example, git, npm, sourceArchive. This is logically, though not actually, equivalent to a mime type.
provider – where entity can be found. Examples include github, npmjs, mavenCentral. The system supports a finite set of providers at any given time.
namespace – a qualifier that helps scope the name of the entity being identified. This typically comes from the context of the type. For example, for NPM packages, it is the scope, for Maven project it would be the groupid, and for GitHub, the login (often org) is used. If an entity does not have a reasonable namespace, a - (hyphen) must be used. That is, the namespace is logically optional but the property must be set.
name – the name of the entity. As with namespace, the name typically comes from the context of the type. So artifactid for Maven, repo name for GitHub, etc.
revision – the instance of the entity being identified. The exact form of the revision depends on the type and provider. In Git, a commit hash is used. In package managers, the typical package version is used. This value is largely uninterpreted by ClearlyDefined and simply either passed to the provider as needed or used as an opaque string in internal keys.

scores

Contains effective and tool score

Harvest Tools

Clearlydefined

{
        "_metadata": {
            "type": "nuget", 
            "url": "cd:/nuget/nuget/-/lamar.microsoft.dependencyinjection/4.4.0", 
            "fetchedAt": "2020-12-04T11:56:33.020Z", 
            "links": {
                "self": {}, 
                "siblings": {}, 
                "licensee": {}, 
                "scancode": {}
            }, 
            "schemaVersion": "1.4.2", 
            "toolVersion": "1.2.2", 
            "processedAt": "2020-12-04T11:56:33.247Z"
        }, 
        "attachments": [
            {
                "path": "clearlydefined/downloaded/LICENSE", 
                "token": "9f20a00ecdcc1e5a7321365f4423ca1269e889de62b3bf213c71d105abc833a3"
            }
        ], 
        "summaryInfo": {
            "k": 208, 
            "count": 11, 
            "hashes": {
                "sha1": "fddbd9daea6a749b32255501b1f5d05ec51e9fea", 
                "sha256": "b5e33401095166de90de0bedccbc8fb4b38260fd927f3726bcf2b024f08a3611"
            }
        }, 
        "files": [
            {
                "path": ".signature.p7s", 
                "hashes": {
                    "sha1": "0b8f5f7a5f057d51480b8375d553e8e40b84ed10", 
                    "sha256": "1062a6d08c958d34d24c38c9f863835b2519495748a7c918287675ff6c6ce08c"
                }
            }, 
        ], 
        "manifest": {
            "@id": "https://api.nuget.org/v3/catalog0/data/2020.11.19.17.58.07/lamar.microsoft.dependencyinjection.4.4.0.json", 
            "@type": [
                "PackageDetails", 
                "catalog:Permalink"
            ], 
            "authors": "Jeremy D. Miller", 
            "catalog:commitId": "75701aa2-7db1-4c03-8ee2-8e94b591fe49", 
            "catalog:commitTimeStamp": "2020-11-19T17:58:07.3181307Z", 
            "created": "2020-11-19T17:56:09.5Z", 
            "description": "Lamar Adapter for ASP.Net Core", 
            "iconUrl": "https://avatars2.githubusercontent.com/u/10048186?v=3&s=200", 
            "id": "Lamar.Microsoft.DependencyInjection", 
            "isPrerelease": false, 
            "lastEdited": "2020-11-19T17:57:40.843Z", 
            "licenseUrl": "https://github.com/JasperFX/lamar/blob/master/LICENSE", 
            "listed": true, 
            "packageHash": "Peije3DisriqrYWrsMwePhaM8uSJTyZ+VFepLRJCu7bHRNC6MyUwEbjfkYqlfHfQeoOCiu8Q7JIuJtwrcdvjCw==", 
            "packageHashAlgorithm": "SHA512", 
            "packageSize": 28384, 
            "projectUrl": "https://jasperfx.github.io/lamar", 
            "published": "2020-11-19T17:56:09.5Z", 
            "requireLicenseAcceptance": false, 
            "verbatimVersion": "4.4.0", 
            "version": "4.4.0", 
            "dependencyGroups": [], 
            "packageEntries": [], 
            "@context": {}
        }, 
        "registryData": {
            "@id": "https://api.nuget.org/v3/registration5-gz-semver2/lamar.microsoft.dependencyinjection/4.4.0.json", 
            "@type": [], 
            "catalogEntry": "https://api.nuget.org/v3/catalog0/data/2020.11.19.17.58.07/lamar.microsoft.dependencyinjection.4.4.0.json", 
            "listed": true, 
            "packageContent": "https://api.nuget.org/v3-flatcontainer/lamar.microsoft.dependencyinjection/4.4.0/lamar.microsoft.dependencyinjection.4.4.0.nupkg", 
            "published": "2020-11-19T17:56:09.5+00:00", 
            "registration": "https://api.nuget.org/v3/registration5-gz-semver2/lamar.microsoft.dependencyinjection/index.json", 
            "@context": {},
            "releaseDate": "2020-11-19T17:56:09.500Z"
}

Contains hashes of files(sha1 and sha256), registryData, Source Info (if source is provided)

Licensee

 {
      "_metadata": {
        "type": "licensee",
        "url": "cd:/sourcearchive/mavencentral/com.azure/azure-spring-data-cosmos/3.2.0",
        "fetchedAt": "2020-12-12T01:12:37.734Z",
        "links": {
          "self": {
            "href": "urn:sourcearchive:mavencentral:com.azure:azure-spring-data-cosmos:revision:3.2.0:tool:licensee:9.13.0",
            "type": "resource"
          },
          "siblings": {
            "href": "urn:sourcearchive:mavencentral:com.azure:azure-spring-data-cosmos:revision:3.2.0:tool:licensee",
            "type": "collection"
          }
        },
        "schemaVersion": "9.13.0",
        "toolVersion": "9.11.0",
        "processedAt": "2020-12-12T01:12:40.508Z"
      },
      "licensee": {
        "version": "9.11.0",
        "parameters": [
          "--json",
          "--no-readme"
        ],
        "output": {
          "contentType": "application/json",
          "content": {
            "licenses": [],
            "matched_files": []
          }
        }
      }
    }

Contains licenses, matched files (content and filename)

Scancode

{
            "_metadata": {
                "type": "scancode", 
                "url": "cd:/sourcearchive/mavencentral/io.quarkus/quarkus-core/1.9.2.Final", 
                "fetchedAt": "2020-12-01T16:38:22.329Z", 
                "links": {}, 
                "schemaVersion": "3.2.2", 
                "toolVersion": "3.0.2", 
                "contentType": "application/json", 
                "releaseDate": "2020-11-30T18:24:04.000Z", 
                "processedAt": "2020-12-01T16:39:01.503Z"
            }, 
            "content": {
                "headers": [], 
                "summary": {
                    "license_expressions": [ {
                                "value": "apache-2.0",
                                "count": 3], 
                    "copyrights": [{
                       "value": "Copyright (c) Hiroyuki Takagi",
                       "count": 1
                                  }], 
                    "holders": [{
                       "value": "Hiroyuki Takagi",
                       "count": 1
                               }], 
                    "authors": [], 
                    "programming_language": [], 
                    "packages": []
                }, 
                "license_clarity_score": {
                    "score": 15, 
                    "has_declared_license_in_key_files": false, 
                    "file_level_license_and_copyright_coverage": 0, 
                    "has_consistent_key_and_file_level_licenses": false, 
                    "is_using_only_spdx_licenses": true, 
                    "has_full_text_for_all_licenses": false
                }, 
                "summary_of_key_files": {

                    "license_expressions": [ {
                                "value": "apache-2.0",
                                "count": 3
                                           }], 
                    "copyrights": [ {
                       "value": "Copyright (c) Hiroyuki Takagi",
                       "count": 1
                                  }], 
                    "holders": [{
                       "value": "Hiroyuki Takagi",
                       "count": 1
                               }], 
                    "authors": [ ], 
                    "programming_language": [ ]
                }, 
                "files": [
                    {
                        "path": "io", 
                        "type": "directory", 
                        "name": "io", 
                        "base_name": "io", 
                        "extension": "", 
                        "size": 0, 
                        "date": null, 
                        "sha1": null, 
                        "md5": null, 
                        "mime_type": null, 
                        "file_type": null, 
                        "programming_language": null, 
                        "is_binary": false, 
                        "is_text": false, 
                        "is_archive": false, 
                        "is_media": false, 
                        "is_source": false, 
                        "is_script": false, 
                        "licenses": [ ], 
                        "license_expressions": [ ], 
                        "holders": [ ], 
                        "copyrights": [ ], 
                        "authors": [ ], 
                        "packages": [ ], 
                        "emails": [ ], 
                        "urls": [ ], 
                        "is_legal": false, 
                        "is_manifest": false, 
                        "is_readme": false, 
                        "is_top_level": true, 
                        "is_key_file": false, 
                        "is_generated": false, 
                        "is_license_text": false, 
                        "files_count": 159, 
                        "dirs_count": 12, 
                        "size_count": 294024, 
                        "scan_errors": [ ]
                    }, 
                    ]
            }
}

Contains Summary (license_expressions, copyrights, holders, authors, programming Language), license clarity score, summary of key files and a separate analysis of every file.

Scancode(per file information)

"path": "pyppeteer-0.2.5/LICENSE",
"type": "file",
"name": "LICENSE",
"base_name": "LICENSE",
"extension": "",
"size": 1166,
"date": "1970-01-01",
"sha1": "b88cc146668fa09ce5aa419bf7022c39d5cf52cb",
"md5": "3782fea6a4e902a0f3ed33554f1bde32",
"mime_type": "text/plain",
"file_type": "ASCII text, with very long lines, with CRLF line terminators",
"programming_language": null,
"is_binary": false,
"is_text": true,
"is_archive": false,
"is_media": false,
"is_source": false,
"is_script": false,
"licenses": [
{"key": "mit",
"score": 99.4,
"name": "MIT License",
"short_name": "MIT License",
"category": "Permissive",
"is_exception": false,
"owner": "MIT",
"homepage_url": "http://opensource.org/licenses/mit-license.php",
"text_url": "http://opensource.org/licenses/mit-license.php",
"reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit",
"spdx_license_key": "MIT",
"spdx_url": "https://spdx.org/licenses/MIT",
"start_line": 2,
"end_line": 10,
"matched_rule": {
"identifier": "mit_160.RULE",
"license_expression": "mit",
"licenses": ["mit"],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"matcher": "3-seq",
"rule_length": 167,
"matched_length": 166,
"match_coverage": 99.4,
"rule_relevance": 100
},
"matched_text": "MIT License\n\nCopyright ([c]) [2017], [Hiroyuki] [Takagi]\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation .......  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE."
{
"key": "apache-2.0",
"score": 50,
"name": "Apache License 2.0",
"short_name": "Apache 2.0",
"category": "Permissive",
"is_exception": false,
"owner": "Apache Software Foundation",
"homepage_url": "http://www.apache.org/licenses/",
"text_url": "http://www.apache.org/licenses/LICENSE-2.0",
"reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key": "Apache-2.0",
"spdx_url": "https://spdx.org/licenses/Apache-2.0",
"start_line": 12,
"end_line": 12,
"matched_rule": {
"identifier": "apache-2.0_3.RULE",
"license_expression": "apache-2.0",
"licenses": [
"apache-2.0"],
"is_license_text": false,
"is_license_notice": false,
"is_license_reference": true,
"is_license_tag": false,
"matcher": "2-aho",
"rule_length": 4,
"matched_length": 4,
"match_coverage": 100,
"rule_relevance": 50
},
"matched_text": "Apache License 2.0."
}
],
"license_expressions": [
"mit",
"apache-2.0"
],
"holders": [
{
"value": "Hiroyuki Takagi",
"start_line": 4,
"end_line": 4
}
],
"copyrights": [
{
"value": "Copyright (c) 2017, Hiroyuki Takagi",
"start_line": 4,
"end_line": 4
}
],
"authors": [],
"packages": [],
"emails": [],
"urls": [],
"is_legal": true,
"is_manifest": false,
"is_readme": false,
"is_top_level": true,
"is_key_file": true,
"is_generated": false,
"is_license_text": true,
"files_count": 0,
"dirs_count": 0,
"size_count": 0,
"scan_errors": []
},