Skip to content

Organized and cleaned up the Code #587

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RISHII-BHARADHWAJ wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Organized and cleaned up the Code #587

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2b5c8e1
Organized and cleaned up Code
Jun 2, 2025
f55b9bb
Fix: ensure all license blocks rendered in HTML output, resolve impor…
RISHII-BHARADHWAJ Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/attributecode/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

from attributecode import ERROR
from attributecode import Error

from attributecode import __version__
"""
API call helpers
"""
Expand Down
8 changes: 5 additions & 3 deletions src/attributecode/attrib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# -*- coding: utf8 -*-

# ============================================================================
# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved.
# Copyright (c) nexB Inc. http://www.nexB.com/ - All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand Down Expand Up @@ -62,7 +62,9 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,

template = jinja2.Template(template)
# Get the current UTC time
utcnow = datetime.datetime.utcnow()
# utcnow = datetime.datetime.utcnow()
# Use timezone-aware UTC datetime to avoid deprecation warning
utcnow = datetime.datetime.now(datetime.timezone.utc)

licenses_list = []
lic_name_expression_list = []
Expand Down Expand Up @@ -154,7 +156,7 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,

rendered = template.render(
abouts=abouts,
common_licenses=COMMON_LICENSES,
common_licenses=[lic.key for lic in licenses_list],
Copy link
Contributor

@chinyeungli chinyeungli Feb 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The COMMON_LICENSES (https://github.com/aboutcode-org/aboutcode-toolkit/blob/develop/src/attributecode/licenses.py) contains commonly encountered licenses in general, not the licenses actually detected within a specific codebase

licenses_list=licenses_list,
utcnow=utcnow,
tkversion=__version__,
Expand Down
3 changes: 2 additions & 1 deletion src/attributecode/cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# -*- coding: utf8 -*-

# ============================================================================
# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved.
# Copyright (c) nexB Inc. http://www.nexB.com/ - All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand Down Expand Up @@ -40,6 +40,7 @@
from attributecode import __about_spec_version__
from attributecode.util import unique
from attributecode import WARNING
from attributecode import Error

from collections import defaultdict
from functools import partial
Expand Down
4 changes: 2 additions & 2 deletions src/attributecode/gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

import saneyaml
Copy link
Contributor

@chinyeungli chinyeungli Feb 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do I need to import saneyaml if it isn’t used?

from posixpath import basename
from posixpath import dirname
from posixpath import exists
Expand All @@ -34,7 +34,7 @@
from attributecode.util import UNC_PREFIX_POSIX
from attributecode.util import load_scancode_json, load_csv, load_json, load_excel
from attributecode.util import strip_inventory_value

from attributecode import __version__

def check_duplicated_columns(location):
"""
Expand Down
94 changes: 28 additions & 66 deletions src/attributecode/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf8 -*-
# ============================================================================
# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved.
# Copyright (c) nexB Inc. http://www.nexB.com/ - All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand Down Expand Up @@ -37,34 +37,32 @@

from license_expression import Licensing
from packageurl import PackageURL

from attributecode import __version__
from attributecode import CRITICAL
from attributecode import ERROR
from attributecode import INFO
from attributecode import WARNING
from attributecode import api
from attributecode import Error
from attributecode import saneyaml
from attributecode import gen
from attributecode import util
from attributecode.transform import write_excel
from attributecode.util import add_unc
from attributecode.util import boolean_fields
from attributecode.util import copy_license_notice_files
from attributecode.util import copy_file
from attributecode.util import csv
from attributecode.util import file_fields
from attributecode.util import filter_errors
from attributecode.util import get_spdx_key_and_lic_key_from_licdb
from attributecode.util import is_valid_name
from attributecode.util import on_windows
from attributecode.util import norm
from attributecode.util import replace_tab_with_spaces
from attributecode.util import wrap_boolean_value
from attributecode.util import UNC_PREFIX
from attributecode.util import ungroup_licenses
from attributecode.util import ungroup_licenses_from_sctk
from attributecode import Error, CRITICAL, ERROR, WARNING, INFO
from attributecode.util import (
Copy link
Contributor

@chinyeungli chinyeungli Feb 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For consistency, it would be better to keep those import statements inline rather than grouped in a set.

add_unc,
boolean_fields,
copy_license_notice_files,
copy_file,
csv,
file_fields,
filter_errors,
get_spdx_key_and_lic_key_from_licdb,
is_valid_name,
on_windows,
norm,
replace_tab_with_spaces,
wrap_boolean_value,
UNC_PREFIX,
ungroup_licenses,
ungroup_licenses_from_sctk,
parse_license_expression,
detect_special_char,
valid_api_url
)

import saneyaml

genereated_tk_version = "# Generated with AboutCode Toolkit Version %s \n\n" % __version__

Expand Down Expand Up @@ -234,9 +232,10 @@ class StringField(Field):
"""

def _validate(self, *args, **kwargs):
errors = super(StringField, self)._validate(*args, ** kwargs)
errors = super(StringField, self)._validate(*args, **kwargs)

no_special_char_field = [
'license_expression', 'license_key', 'license_name', 'declared_license_expression', 'other_license_expression ']
'license_expression', 'license_key', 'license_name', 'declared_license_expression', 'other_license_expression']
name = self.name
if name in no_special_char_field:
val = self.value
Expand Down Expand Up @@ -2145,40 +2144,3 @@ def convert_spdx_expression_to_lic_expression(spdx_key, spdx_lic_dict):
return value


def parse_license_expression(lic_expression):
licensing = Licensing()
lic_list = []
invalid_lic_exp = ''
special_char = detect_special_char(lic_expression)
if not special_char:
# Parse the license expression and save it into a list
try:
lic_list = licensing.license_keys(lic_expression)
except:
invalid_lic_exp = lic_expression
return special_char, lic_list, invalid_lic_exp


def detect_special_char(expression):
not_support_char = [
'!', '@', '#', '$', '^', '&', '*', '=', '{', '}',
'|', '[', ']', '\\', ':', ';', '<', '>', '?', ',', '/']
special_character = []
for char in not_support_char:
if char in expression:
special_character.append(char)
return special_character


def valid_api_url(api_url):
try:
response = get(api_url)
# The 403 error code is expected if the api_url is pointing to DJE as no
# API key is provided. The 200 status code represent connection success
# to scancode's LicenseDB. All other exception yield to invalid api_url
if response.status_code == 403 or response.status_code == 200:
return True
else:
return False
except:
return False
8 changes: 6 additions & 2 deletions src/attributecode/transform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,12 @@
from attributecode import CRITICAL
from attributecode import Error
from attributecode import saneyaml
from attributecode.util import csv
from attributecode.util import replace_tab_with_spaces
from attributecode.util import (
Copy link
Contributor

@chinyeungli chinyeungli Feb 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For consistency, it would be better to keep those import statements inline rather than grouped in a set.

csv,
replace_tab_with_spaces,
parse_license_expression, # canonical implementation from util.py
detect_special_char, # canonical implementation from util.py
)


def transform_csv(location):
Expand Down
46 changes: 46 additions & 0 deletions src/attributecode/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
from attributecode import CRITICAL
from attributecode import WARNING
from attributecode import Error
from attributecode import __version__
from attributecode import Error
Copy link
Contributor

@chinyeungli chinyeungli Feb 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It’s a duplicated import.


on_windows = "win32" in sys.platform

Expand Down Expand Up @@ -273,6 +275,50 @@ def get_spdx_key_and_lic_key_from_licdb():

return lic_dict

"""
Canonical implementation of license expression parsing and special character detection.
Import and use these from util.py everywhere in the codebase to avoid duplication.
"""
def parse_license_expression(lic_expression):
from license_expression import Licensing
licensing = Licensing()
lic_list = []
invalid_lic_exp = ''
special_char = detect_special_char(lic_expression)
if not special_char:
# Parse the license expression and save it into a list
try:
lic_list = licensing.license_keys(lic_expression)
except Exception:
invalid_lic_exp = lic_expression
return special_char, lic_list, invalid_lic_exp

def detect_special_char(expression):
not_support_char = [
'!', '@', '#', '$', '^', '&', '*', '=', '{', '}',
'|', '[', ']', '\\', ':', ';', '<', '>', '?', ',', '/']
special_character = []
if not isinstance(expression, str):
return special_character
for char in not_support_char:
if char in expression:
special_character.append(char)
return special_character


def valid_api_url(api_url):
try:
response = get(api_url)
# The 403 error code is expected if the api_url is pointing to DJE as no
# API key is provided. The 200 status code represent connection success
# to scancode's LicenseDB. All other exception yield to invalid api_url
if response.status_code == 403 or response.status_code == 200:
return True
else:
return False
except:
return False


def get_relative_path(base_loc, full_loc):
"""
Expand Down
53 changes: 42 additions & 11 deletions tests/test_attrib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#!/usr/bin/env python

# -*- coding: utf8 -*-

# ============================================================================
# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved.
# Copyright (c) nexB Inc. http://www.nexB.com/ - All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand Down Expand Up @@ -175,8 +176,14 @@ def test_scancode_input_min_score_0(self):
# expected doesn't work well, it works after removed all the newline and spaces
# assert expected == result
# assert expected.splitlines(False) == result.splitlines(False)
assert expected.replace('\n', '').replace(' ', '').replace(
'\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '')
actual = result.replace('\n', '').replace(' ', '').replace('\t', '')
exp = expected.replace('\n', '').replace(' ', '').replace('\t', '')
if actual != exp:
print('---EXPECTED---')
print(expected)
print('---ACTUAL---')
print(result)
assert exp == actual

def test_scancode_input_min_score_100(self):
test_file = get_test_loc(
Expand Down Expand Up @@ -207,8 +214,14 @@ def test_scancode_input_min_score_100(self):
# expected doesn't work well, it works after removed all the newline and spaces
# assert expected == result
# assert expected.splitlines(False) == result.splitlines(False)
assert expected.replace('\n', '').replace(' ', '').replace(
'\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '')
actual = result.replace('\n', '').replace(' ', '').replace('\t', '')
exp = expected.replace('\n', '').replace(' ', '').replace('\t', '')
if actual != exp:
print('---EXPECTED---')
print(expected)
print('---ACTUAL---')
print(result)
assert exp == actual

def test_scancode_input_dup_lic(self):
test_file = get_test_loc('test_attrib/scancode_input/sc-dup-lic.json')
Expand Down Expand Up @@ -238,8 +251,14 @@ def test_scancode_input_dup_lic(self):
# expected doesn't work well, it works after removed all the newline and spaces
# assert expected == result
# assert expected.splitlines(False) == result.splitlines(False)
assert expected.replace('\n', '').replace(' ', '').replace(
'\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '')
actual = result.replace('\n', '').replace(' ', '').replace('\t', '')
exp = expected.replace('\n', '').replace(' ', '').replace('\t', '')
if actual != exp:
print('---EXPECTED---')
print(expected)
print('---ACTUAL---')
print(result)
assert exp == actual

def test_scancode_input_dup_lic_match(self):
test_file = get_test_loc(
Expand Down Expand Up @@ -272,8 +291,14 @@ def test_scancode_input_dup_lic_match(self):
# expected doesn't work well, it works after removed all the newline and spaces
# assert expected == result
# assert expected.splitlines(False) == result.splitlines(False)
assert expected.replace('\n', '').replace(' ', '').replace(
'\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '')
actual = result.replace('\n', '').replace(' ', '').replace('\t', '')
exp = expected.replace('\n', '').replace(' ', '').replace('\t', '')
if actual != exp:
print('---EXPECTED---')
print(expected)
print('---ACTUAL---')
print(result)
assert exp == actual

def test_scancode_input_multi_lic(self):
test_file = get_test_loc(
Expand Down Expand Up @@ -304,8 +329,14 @@ def test_scancode_input_multi_lic(self):
# expected doesn't work well, it works after removed all the newline and spaces
# assert expected == result
# assert expected.splitlines(False) == result.splitlines(False)
assert expected.replace('\n', '').replace(' ', '').replace(
'\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '')
actual = result.replace('\n', '').replace(' ', '').replace('\t', '')
exp = expected.replace('\n', '').replace(' ', '').replace('\t', '')
if actual != exp:
print('---EXPECTED---')
print(expected)
print('---ACTUAL---')
print(result)
assert exp == actual

def test_generate_with_csv(self):
test_file = get_test_loc(
Expand Down
Loading
Loading