From 912de363f4658346f05211ce62d3d063fc61f6c9 Mon Sep 17 00:00:00 2001 From: tribta Date: Fri, 28 Jul 2017 20:52:56 +0100 Subject: [PATCH 1/6] Using __doc__ to add Python signatures to the docs. --- doc/CMakeLists.txt | 24 +++- doc/tools/html_functions.py | 209 +++++++++++++++++++++++++++++++++ doc/tools/python_signatures.py | 34 ++++++ 3 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 doc/tools/html_functions.py create mode 100644 doc/tools/python_signatures.py diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 3d6e3950d1..7eaf62e379 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -205,7 +205,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) list(APPEND js_tutorials_assets_deps "${f}" "${opencv_tutorial_html_dir}/${fname}") endforeach() - add_custom_target(doxygen + add_custom_target(doxygen_cpp COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} DEPENDS ${doxyfile} ${rootfile} ${bibfile} ${deps} ${js_tutorials_assets_deps} ) @@ -214,6 +214,28 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) COMPONENT "docs" OPTIONAL ) + if(BUILD_opencv_python2) + add_custom_target(doxygen_python + COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" + DEPENDS doxygen_cpp opencv_python2 + ) + add_custom_target(doxygen + DEPENDS doxygen_cpp doxygen_python + ) + elseif(BUILD_opencv_python3) + add_custom_target(doxygen_python + COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" + DEPENDS doxygen_cpp opencv_python3 + ) + add_custom_target(doxygen + DEPENDS doxygen_cpp doxygen_python + ) + else() + add_custom_target(doxygen + DEPENDS doxygen_cpp + ) + endif() + # Alias to build/install docs only add_custom_target(install_docs DEPENDS doxygen diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py new file mode 100644 index 0000000000..bf9577bad4 --- /dev/null +++ b/doc/tools/html_functions.py @@ -0,0 +1,209 @@ +import logging +import os +import codecs +import cv2 + + +try: + from bs4 import BeautifulSoup +except ImportError: + raise ImportError('Error: ' + 'Install BeautifulSoup (bs4) for adding' + ' Python & Java signatures documentation') + + +def is_not_module_link(tmp_link): + """ Checks if a link belongs to a c++ method """ + if tmp_link is None: + return True + if "group" not in tmp_link: + return True + if "#" in tmp_link: + return True + return False + + +def get_links_list(tmp_soup, filter_links): + """ Get a list of links from a soup """ + tmp_href_list = [] + for tmp_link in tmp_soup.findAll('a'): + tmp_href = tmp_link.get('href') + if filter_links: + if is_not_module_link(tmp_href): + continue + tmp_href_list.append(tmp_href) + return tmp_href_list + + +def load_html_file(file_dir): + """ Uses BeautifulSoup to load an html """ + with open(file_dir) as fp: + tmp_soup = BeautifulSoup(fp, 'html.parser') + return tmp_soup + + +def add_item(tmp_soup, new_row, is_parameter, text): + """ Adds a new html tag for the table with the signature """ + new_item = tmp_soup.new_tag('td') + if is_parameter: + new_item = tmp_soup.new_tag('td', **{'class': 'paramname'}) + new_item.append(text) + new_row.append(new_item) + return new_row + + +def get_text_between_substrings(sig, begin_char, end_char): + return sig.partition(begin_char)[-1].rpartition(end_char)[0] + + +def add_signature_to_table(tmp_soup, new_row, signature, function_name, language, ident): + """ Add a signature to an html table""" + if ident: + new_item = tmp_soup.new_tag('td', style="padding-left: 0.5cm;") + else: + new_item = tmp_soup.new_tag('td') + + if "-> None" in signature: + pass + elif "->" in signature: + new_item.append(signature.split("->", 1)[1] + ' =') + new_row.append(new_item) + + if "Python" in language: + function_name = "cv2." + function_name + elif "Java" in language: + # get word before function_name (= output) + str_before_bracket = signature.split('(', 1)[0] + list_of_words = str_before_bracket.split() + output = list_of_words[len(list_of_words) - 2] + new_item.append(output + " ") + new_row.append(new_item) + + new_row = add_item(tmp_soup, new_row, False, function_name + '(') + new_row = add_item(tmp_soup, new_row, True, get_text_between_substrings(signature, "(", ")")) + new_row = add_item(tmp_soup, new_row, False, ')') + return new_row + + +def new_line(tmp_soup, tmp_table, new_row): + """ Adds a new line to the html table """ + tmp_table.append(new_row) + new_row = tmp_soup.new_tag('tr') + return new_row + + +def add_bolded(tmp_soup, new_row, text): + """ Adds bolded text to the table """ + new_item = tmp_soup.new_tag('th', style="text-align:left") + new_item.append(text) + new_row.append(new_item) + return new_row + + +def append_table_to(cpp_table, tmp_soup, language, signature, function_name): + """ Insert the new Python / Java table after the current html c++ table """ + if signature != "": + tmp_table = tmp_soup.new_tag('table') + new_row = tmp_soup.new_tag('tr') + new_row = add_bolded(tmp_soup, new_row, language) + ident = False + + if len(signature) > 120: + new_row = new_line(tmp_soup, tmp_table, new_row) + ident = True + + if " or " in signature: + ident = True + for tmp_sig in signature.split(" or "): + new_row = new_line(tmp_soup, tmp_table, new_row) + new_row = add_signature_to_table(tmp_soup, new_row, tmp_sig, function_name, language, ident) + new_row = new_line(tmp_soup, tmp_table, new_row) + else: + new_row = add_signature_to_table(tmp_soup, new_row, signature, function_name, language, ident) + tmp_table.append(new_row) + + cpp_table.insert_after(tmp_table) + return cpp_table + + +def add_signatures(tmp_soup, tmp_dir, ADD_JAVA, ADD_PYTHON, module_name): + """ Add signatures to the current soup and rewrite the html file""" + logging.debug(tmp_dir) + sign_counter = 0 + python_sign_counter = 0 + java_sign_counter = 0 + + if ADD_JAVA: + functions_file = "java_doc_txts/" + module_name + "/functions.txt" + if os.path.exists(functions_file): + with open(functions_file, 'r') as f: + java_signatures = f.read().split("\n") + else: + ADD_JAVA = False # This C++ module (module_name) may not exist in Java + + # the HTML tag & class being used to find functions + for function in tmp_soup.findAll("h2", {"class": "memtitle"}): + function_name = function.getText() + if os.name == 'nt': # if Windows + function_name = function_name.encode("ascii","ignore").decode() + + # all functions have () in it's name + if "()" not in function_name: + continue + + if "[" in function_name: + if "[1/" in function_name: + function_name = function_name.replace(' ', '')[:-7] + else: + continue + else: + function_name = function_name.replace(' ', '')[:-2] + sign_counter += 1 + + # if not Windows computer + if os.name != 'nt': + function_name = function_name.replace(' ', '')[2:] + + cpp_table = function.findNext('table') + + if ADD_PYTHON: + try: + print(function_name) + method = getattr(cv2, str(function_name)) + description = str(method.__doc__).split("\n") + signature = "" + is_first_sig = True + for line in description: + if line.startswith(".") or line == "": + continue + else: + if is_first_sig: + signature += line + is_first_sig = False + else: + signature += " or " + line + + cpp_table = append_table_to(cpp_table, tmp_soup, "Python:", signature, function_name) + python_sign_counter += 1 + except AttributeError: + continue + + if ADD_JAVA: + for signature in java_signatures: + if function_name in signature: + append_table_to(cpp_table, tmp_soup, "Java:", signature, function_name) + java_sign_counter += 1 + break + + tmp_str = str(tmp_soup) + if os.name == 'nt': # if Windows + with open(tmp_dir, "wb") as tmp_file: + tmp_file.write(tmp_str.encode("ascii","ignore")) + else: + with open(tmp_dir, "w") as tmp_file: + tmp_file.write(tmp_str) + + logging.debug("Added [" + str(python_sign_counter) + \ + "/" + str(sign_counter) + "] Python signatures") + logging.debug("Added [" + str(java_sign_counter) + \ + "/" + str(sign_counter) + "] Java signatures") diff --git a/doc/tools/python_signatures.py b/doc/tools/python_signatures.py new file mode 100644 index 0000000000..34ea3229bd --- /dev/null +++ b/doc/tools/python_signatures.py @@ -0,0 +1,34 @@ +""" +This code adds Python signatures to the docs. + +TODO: +* clarify when there are several C++ signatures corresponding to a single Python function. + i.e: calcHist(): + http://docs.opencv.org/3.2.0/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d +* clarify special case: + http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 +""" +import re +import sys +import html_functions + +ADD_JAVA = False +ADD_PYTHON = True +ROOT_DIR = sys.argv[1] + +soup = html_functions.load_html_file(ROOT_DIR + "index.html") +href_list = html_functions.get_links_list(soup, True) + +for link in href_list: + # add python signatures to the module + soup = html_functions.load_html_file(ROOT_DIR + link) + sub_href_list = html_functions.get_links_list(soup, True) + module_name = html_functions.get_text_between_substrings(link, "group__", ".html") + html_functions.add_signatures(soup, ROOT_DIR + link, ADD_JAVA, ADD_PYTHON, module_name) + + # add python signatures to the sub-modules + link = re.sub(r"group__.+html", "", link) + for sub_link in sub_href_list: + tmp_dir = ROOT_DIR + link + sub_link + soup = html_functions.load_html_file(tmp_dir) + html_functions.add_signatures(soup, tmp_dir, ADD_JAVA, ADD_PYTHON, module_name) From d715badbde9c5a4fa739e48db484212c3d772fe6 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Sep 2017 18:55:01 +0300 Subject: [PATCH 2/6] doc: update Python signatures generation - drop dependency on 'import cv2', use pyopencv_signatures.json instead - try to make generator idempotent --- doc/CMakeLists.txt | 12 ++-- doc/tools/html_functions.py | 118 ++++++++++++++------------------- doc/tools/python_signatures.py | 30 ++++++++- 3 files changed, 85 insertions(+), 75 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 7eaf62e379..741c4a093e 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -205,10 +205,14 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) list(APPEND js_tutorials_assets_deps "${f}" "${opencv_tutorial_html_dir}/${fname}") endforeach() + set(doxygen_result "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/index.html") + add_custom_target(doxygen_cpp COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} DEPENDS ${doxyfile} ${rootfile} ${bibfile} ${deps} ${js_tutorials_assets_deps} + COMMENT "Generate Doxygen documentation" ) + install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html DESTINATION "${OPENCV_DOC_INSTALL_PATH}" COMPONENT "docs" OPTIONAL @@ -216,16 +220,16 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) if(BUILD_opencv_python2) add_custom_target(doxygen_python - COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" - DEPENDS doxygen_cpp opencv_python2 + COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" + DEPENDS "${doxygen_result}" gen_opencv_python2 ) add_custom_target(doxygen DEPENDS doxygen_cpp doxygen_python ) elseif(BUILD_opencv_python3) add_custom_target(doxygen_python - COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" - DEPENDS doxygen_cpp opencv_python3 + COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" + DEPENDS "${doxygen_result}" gen_opencv_python3 ) add_custom_target(doxygen DEPENDS doxygen_cpp doxygen_python diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index bf9577bad4..215a0581d4 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -1,10 +1,11 @@ +from __future__ import print_function import logging import os import codecs -import cv2 - +from pprint import pprint try: + import bs4 from bs4 import BeautifulSoup except ImportError: raise ImportError('Error: ' @@ -63,14 +64,12 @@ def add_signature_to_table(tmp_soup, new_row, signature, function_name, language else: new_item = tmp_soup.new_tag('td') - if "-> None" in signature: - pass - elif "->" in signature: - new_item.append(signature.split("->", 1)[1] + ' =') + if str(signature.get('ret', None)) != "None": + new_item.append(signature.get('ret') + ' =') new_row.append(new_item) if "Python" in language: - function_name = "cv2." + function_name + pass # function_name = "cv2." + function_name elif "Java" in language: # get word before function_name (= output) str_before_bracket = signature.split('(', 1)[0] @@ -79,8 +78,8 @@ def add_signature_to_table(tmp_soup, new_row, signature, function_name, language new_item.append(output + " ") new_row.append(new_item) - new_row = add_item(tmp_soup, new_row, False, function_name + '(') - new_row = add_item(tmp_soup, new_row, True, get_text_between_substrings(signature, "(", ")")) + new_row = add_item(tmp_soup, new_row, False, signature.get('name', function_name) + '(') + new_row = add_item(tmp_soup, new_row, True, signature['arg']) new_row = add_item(tmp_soup, new_row, False, ')') return new_row @@ -100,98 +99,81 @@ def add_bolded(tmp_soup, new_row, text): return new_row -def append_table_to(cpp_table, tmp_soup, language, signature, function_name): +def create_description(tmp_soup, language, signatures, function_name): """ Insert the new Python / Java table after the current html c++ table """ - if signature != "": - tmp_table = tmp_soup.new_tag('table') - new_row = tmp_soup.new_tag('tr') - new_row = add_bolded(tmp_soup, new_row, language) - ident = False + assert signatures + tmp_table = tmp_soup.new_tag('table') + new_row = tmp_soup.new_tag('tr') + new_row = add_bolded(tmp_soup, new_row, language) + ident = False - if len(signature) > 120: - new_row = new_line(tmp_soup, tmp_table, new_row) - ident = True + new_row = new_line(tmp_soup, tmp_table, new_row) + ident = True - if " or " in signature: - ident = True - for tmp_sig in signature.split(" or "): - new_row = new_line(tmp_soup, tmp_table, new_row) - new_row = add_signature_to_table(tmp_soup, new_row, tmp_sig, function_name, language, ident) - new_row = new_line(tmp_soup, tmp_table, new_row) - else: - new_row = add_signature_to_table(tmp_soup, new_row, signature, function_name, language, ident) - tmp_table.append(new_row) + for s in signatures: + new_row = new_line(tmp_soup, tmp_table, new_row) + new_row = add_signature_to_table(tmp_soup, new_row, s, function_name, language, ident) + new_row = new_line(tmp_soup, tmp_table, new_row) - cpp_table.insert_after(tmp_table) - return cpp_table + return tmp_table -def add_signatures(tmp_soup, tmp_dir, ADD_JAVA, ADD_PYTHON, module_name): +def add_signatures(tmp_soup, tmp_dir, module_name, config): """ Add signatures to the current soup and rewrite the html file""" + logging.debug(tmp_dir) sign_counter = 0 python_sign_counter = 0 java_sign_counter = 0 - if ADD_JAVA: + if config.ADD_JAVA: functions_file = "java_doc_txts/" + module_name + "/functions.txt" if os.path.exists(functions_file): with open(functions_file, 'r') as f: java_signatures = f.read().split("\n") else: - ADD_JAVA = False # This C++ module (module_name) may not exist in Java + config.ADD_JAVA = False # This C++ module (module_name) may not exist in Java # the HTML tag & class being used to find functions for function in tmp_soup.findAll("h2", {"class": "memtitle"}): - function_name = function.getText() - if os.name == 'nt': # if Windows - function_name = function_name.encode("ascii","ignore").decode() + function_name = None + for c in function.contents: + if isinstance(c, bs4.element.NavigableString): + fn = str(c).encode("ascii","ignore").decode().strip() + if not fn.endswith('()'): # all functions have () in it's name + # enums, structures, etc + continue + function_name = fn[:-2] - # all functions have () in it's name - if "()" not in function_name: + if not function_name: continue - if "[" in function_name: - if "[1/" in function_name: - function_name = function_name.replace(' ', '')[:-7] - else: - continue - else: - function_name = function_name.replace(' ', '')[:-2] sign_counter += 1 - # if not Windows computer - if os.name != 'nt': - function_name = function_name.replace(' ', '')[2:] - cpp_table = function.findNext('table') - if ADD_PYTHON: - try: + if config.ADD_PYTHON: + signatures = config.python_signatures.get("cv::" + str(function_name), None) + if signatures: print(function_name) - method = getattr(cv2, str(function_name)) - description = str(method.__doc__).split("\n") - signature = "" - is_first_sig = True - for line in description: - if line.startswith(".") or line == "": - continue - else: - if is_first_sig: - signature += line - is_first_sig = False - else: - signature += " or " + line - cpp_table = append_table_to(cpp_table, tmp_soup, "Python:", signature, function_name) + description = create_description(tmp_soup, "Python:", signatures, function_name) + description['class'] = 'python_language' + old = cpp_table.next_sibling + if old.name != 'table': + old = None + elif not 'python_language' in old.get('class', []): + old = None + if old is None: + cpp_table.insert_after(description) + else: + old.replace_with(description) python_sign_counter += 1 - except AttributeError: - continue - if ADD_JAVA: + if config.ADD_JAVA: for signature in java_signatures: if function_name in signature: - append_table_to(cpp_table, tmp_soup, "Java:", signature, function_name) + create_description(cpp_table, tmp_soup, "Java:", signature, function_name) java_sign_counter += 1 break diff --git a/doc/tools/python_signatures.py b/doc/tools/python_signatures.py index 34ea3229bd..6908de278a 100644 --- a/doc/tools/python_signatures.py +++ b/doc/tools/python_signatures.py @@ -8,13 +8,37 @@ TODO: * clarify special case: http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 """ +from __future__ import print_function +import os import re import sys -import html_functions +import logging + +loglevel=os.environ.get("LOGLEVEL", None) +if loglevel: + logging.basicConfig(level=loglevel) ADD_JAVA = False ADD_PYTHON = True ROOT_DIR = sys.argv[1] +PYTHON_SIGNATURES_FILE = sys.argv[2] + +import json +python_signatures = dict() +with open(PYTHON_SIGNATURES_FILE, "rt") as f: + python_signatures = json.load(f) + print("Loaded Python signatures: %d" % len(python_signatures)) + +class Configuration(): + def __init__(self): + self.ADD_PYTHON = ADD_PYTHON + self.python_signatures = python_signatures + self.ADD_JAVA = ADD_JAVA + +config = Configuration() + + +import html_functions soup = html_functions.load_html_file(ROOT_DIR + "index.html") href_list = html_functions.get_links_list(soup, True) @@ -24,11 +48,11 @@ for link in href_list: soup = html_functions.load_html_file(ROOT_DIR + link) sub_href_list = html_functions.get_links_list(soup, True) module_name = html_functions.get_text_between_substrings(link, "group__", ".html") - html_functions.add_signatures(soup, ROOT_DIR + link, ADD_JAVA, ADD_PYTHON, module_name) + html_functions.add_signatures(soup, ROOT_DIR + link, module_name, config) # add python signatures to the sub-modules link = re.sub(r"group__.+html", "", link) for sub_link in sub_href_list: tmp_dir = ROOT_DIR + link + sub_link soup = html_functions.load_html_file(tmp_dir) - html_functions.add_signatures(soup, tmp_dir, ADD_JAVA, ADD_PYTHON, module_name) + html_functions.add_signatures(soup, tmp_dir, module_name, config) From 6174f627102edbfd0f89a65e91376b9f06148c96 Mon Sep 17 00:00:00 2001 From: tribta Date: Fri, 17 Nov 2017 12:56:47 -0500 Subject: [PATCH 3/6] Sample for functions --- doc/CMakeLists.txt | 6 +- doc/tools/add_signatures.py | 101 +++++++++++++ doc/tools/doxygen_scan.py | 51 +++++++ doc/tools/html_functions.py | 254 +++++++++++++++------------------ doc/tools/python_signatures.py | 58 -------- 5 files changed, 270 insertions(+), 200 deletions(-) create mode 100644 doc/tools/add_signatures.py create mode 100644 doc/tools/doxygen_scan.py delete mode 100644 doc/tools/python_signatures.py diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 741c4a093e..d79b86c36b 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -205,7 +205,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) list(APPEND js_tutorials_assets_deps "${f}" "${opencv_tutorial_html_dir}/${fname}") endforeach() - set(doxygen_result "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/index.html") + set(doxygen_result "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/modules.html") add_custom_target(doxygen_cpp COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} @@ -220,7 +220,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) if(BUILD_opencv_python2) add_custom_target(doxygen_python - COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" + COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python2 ) add_custom_target(doxygen @@ -228,7 +228,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) ) elseif(BUILD_opencv_python3) add_custom_target(doxygen_python - COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/python_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" + COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python3 ) add_custom_target(doxygen diff --git a/doc/tools/add_signatures.py b/doc/tools/add_signatures.py new file mode 100644 index 0000000000..05c029de25 --- /dev/null +++ b/doc/tools/add_signatures.py @@ -0,0 +1,101 @@ +""" +This code adds Python/Java signatures to the docs. + +TODO: Do the same thing for Java +* using javadoc/ get all the methods/classes/constants to a json file + +TODO: +* clarify when there are several C++ signatures corresponding to a single Python function. + i.e: calcHist(): + http://docs.opencv.org/3.2.0/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d +* clarify special case: + http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 +""" +from __future__ import print_function +import os +import re +import sys +import logging +import html_functions +import doxygen_scan + +loglevel=os.environ.get("LOGLEVEL", None) +if loglevel: + logging.basicConfig(level=loglevel) + + +ROOT_DIR = sys.argv[1] +PYTHON_SIGNATURES_FILE = sys.argv[2] +JAVA_PYTHON = sys.argv[3] + +ADD_JAVA = False +ADD_PYTHON = False +if JAVA_PYTHON == "python": + ADD_PYTHON = True + +import json +python_signatures = dict() +with open(PYTHON_SIGNATURES_FILE, "rt") as f: + python_signatures = json.load(f) + print("Loaded Python signatures: %d" % len(python_signatures)) + +# only name -> class +# name and ret -> constant +# name, ret, arg-> function / class method + +class Configuration(): + def __init__(self): + self.ADD_PYTHON = ADD_PYTHON + self.python_signatures = python_signatures + self.ADD_JAVA = ADD_JAVA + +config = Configuration() + +import xml.etree.ElementTree as ET +root = ET.parse(ROOT_DIR + 'opencv.tag') +files_dict = dict() + +# constants and function from opencv.tag +namespaces = root.findall("./compound[@kind='namespace']") +#print("Found {} namespaces".format(len(namespaces))) +for ns in namespaces: + ns_name = ns.find("./name").text + #print('NS: {}'.format(ns_name)) + + files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict) + files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict) + +# class methods from opencv.tag +classes = root.findall("./compound[@kind='class']") +#print("Found {} classes".format(len(classes))) +for c in classes: + c_name = c.find("./name").text + name = ns_name + '::' + c_name + file = c.find("./filename").text + #print('Class: {} => {}'.format(name, file)) + files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict) + +# test +for file in files_dict: + soup = html_functions.load_html_file(ROOT_DIR + file) + if file == "d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html": + anchor_list = files_dict[file] + counter = 0 + anchor_tmp_list = [] + for anchor in anchor_list: + counter += 1 + # if the next anchor shares the same C++ name (= same method/function), join them together + if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname: + anchor_tmp_list.append(anchor) + continue + else: + anchor_tmp_list.append(anchor) + # check if extists a python equivalent signature + for signature in python_signatures: # signature is a key with the C++ name + if signature == anchor.cppname: # if available name in python + # they should also have the same type + soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup) + #print(signature) + # reset anchor temporary list + anchor_tmp_list[:] = [] + html_functions.update_html(ROOT_DIR + file, soup) diff --git a/doc/tools/doxygen_scan.py b/doc/tools/doxygen_scan.py new file mode 100644 index 0000000000..4567723302 --- /dev/null +++ b/doc/tools/doxygen_scan.py @@ -0,0 +1,51 @@ +class Anchor(object): + anchor = "" + type = "" + cppname = "" + + def __init__(self, anchor, type, cppname): + self.anchor = anchor + self.type = type + self.cppname = cppname + +def add_to_file(files_dict, file, anchor): + if file in files_dict: + # if that file already exists as a key in the dictionary + files_dict[file].append(anchor) + else: + files_dict[file] = [anchor] + return files_dict + + +def scan_namespace_constants(ns, ns_name, files_dict): + constants = ns.findall("./member[@kind='enumvalue']") + for c in constants: + c_name = c.find("./name").text + name = ns_name + '::' + c_name + file = c.find("./anchorfile").text + anchor = c.find("./anchor").text + #print(' CONST: {} => {}#{}'.format(name, file, anchor)) + files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name)) + return files_dict + +def scan_namespace_functions(ns, ns_name, files_dict): + functions = ns.findall("./member[@kind='function']") + for f in functions: + f_name = f.find("./name").text + name = ns_name + '::' + f_name + file = f.find("./anchorfile").text + anchor = f.find("./anchor").text + #print(' FN: {} => {}#{}'.format(name, file, anchor)) + files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name)) + return files_dict + +def scan_class_methods(c, c_name, files_dict): + methods = c.findall("./member[@kind='function']") + for m in methods: + m_name = m.find("./name").text + name = c_name + '::' + m_name + file = m.find("./anchorfile").text + anchor = m.find("./anchor").text + #print(' Method: {} => {}#{}'.format(name, file, anchor)) + files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name)) + return files_dict diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index 215a0581d4..ca3e096cde 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -1,7 +1,6 @@ from __future__ import print_function import logging import os -import codecs from pprint import pprint try: @@ -12,180 +11,157 @@ except ImportError: 'Install BeautifulSoup (bs4) for adding' ' Python & Java signatures documentation') - -def is_not_module_link(tmp_link): - """ Checks if a link belongs to a c++ method """ - if tmp_link is None: - return True - if "group" not in tmp_link: - return True - if "#" in tmp_link: - return True - return False - - -def get_links_list(tmp_soup, filter_links): - """ Get a list of links from a soup """ - tmp_href_list = [] - for tmp_link in tmp_soup.findAll('a'): - tmp_href = tmp_link.get('href') - if filter_links: - if is_not_module_link(tmp_href): - continue - tmp_href_list.append(tmp_href) - return tmp_href_list - - def load_html_file(file_dir): """ Uses BeautifulSoup to load an html """ with open(file_dir) as fp: - tmp_soup = BeautifulSoup(fp, 'html.parser') - return tmp_soup + soup = BeautifulSoup(fp, 'html.parser') + return soup - -def add_item(tmp_soup, new_row, is_parameter, text): +def add_item(soup, new_row, is_parameter, text): """ Adds a new html tag for the table with the signature """ - new_item = tmp_soup.new_tag('td') + new_item = soup.new_tag('td') if is_parameter: - new_item = tmp_soup.new_tag('td', **{'class': 'paramname'}) + new_item = soup.new_tag('td', **{'class': 'paramname'}) new_item.append(text) new_row.append(new_item) - return new_row + return new_row, soup - -def get_text_between_substrings(sig, begin_char, end_char): - return sig.partition(begin_char)[-1].rpartition(end_char)[0] - - -def add_signature_to_table(tmp_soup, new_row, signature, function_name, language, ident): +def add_signature_to_table(soup, tmp_row, signature, language): """ Add a signature to an html table""" - if ident: - new_item = tmp_soup.new_tag('td', style="padding-left: 0.5cm;") - else: - new_item = tmp_soup.new_tag('td') + new_item = soup.new_tag('td', style="padding-left: 0.5cm;") if str(signature.get('ret', None)) != "None": new_item.append(signature.get('ret') + ' =') - new_row.append(new_item) + tmp_row.append(new_item) - if "Python" in language: - pass # function_name = "cv2." + function_name - elif "Java" in language: - # get word before function_name (= output) - str_before_bracket = signature.split('(', 1)[0] - list_of_words = str_before_bracket.split() - output = list_of_words[len(list_of_words) - 2] - new_item.append(output + " ") - new_row.append(new_item) - - new_row = add_item(tmp_soup, new_row, False, signature.get('name', function_name) + '(') - new_row = add_item(tmp_soup, new_row, True, signature['arg']) - new_row = add_item(tmp_soup, new_row, False, ')') - return new_row + tmp_row, soup = add_item(soup, tmp_row, False, "cv2." + signature.get('name', None) + '(') + tmp_row, soup = add_item(soup, tmp_row, True, signature['arg']) + tmp_row, soup = add_item(soup, tmp_row, False, ')') + return tmp_row, soup -def new_line(tmp_soup, tmp_table, new_row): +def new_line(soup, tmp_table, new_row): """ Adds a new line to the html table """ tmp_table.append(new_row) - new_row = tmp_soup.new_tag('tr') - return new_row + new_row = soup.new_tag('tr') + return new_row, soup -def add_bolded(tmp_soup, new_row, text): +def add_bolded(soup, new_row, text): """ Adds bolded text to the table """ - new_item = tmp_soup.new_tag('th', style="text-align:left") + new_item = soup.new_tag('th', style="text-align:left") new_item.append(text) new_row.append(new_item) - return new_row + return new_row, soup -def create_description(tmp_soup, language, signatures, function_name): +def create_description(soup, language, signatures): """ Insert the new Python / Java table after the current html c++ table """ assert signatures - tmp_table = tmp_soup.new_tag('table') - new_row = tmp_soup.new_tag('tr') - new_row = add_bolded(tmp_soup, new_row, language) - ident = False - - new_row = new_line(tmp_soup, tmp_table, new_row) - ident = True - + tmp_table = soup.new_tag('table') + new_row = soup.new_tag('tr') + new_row, soup = add_bolded(soup, new_row, language) + new_row, soup = new_line(soup, tmp_table, new_row) for s in signatures: - new_row = new_line(tmp_soup, tmp_table, new_row) - new_row = add_signature_to_table(tmp_soup, new_row, s, function_name, language, ident) - new_row = new_line(tmp_soup, tmp_table, new_row) - - return tmp_table + new_row, soup = new_line(soup, tmp_table, new_row) + new_row, soup = add_signature_to_table(soup, new_row, s, language) + new_row, soup = new_line(soup, tmp_table, new_row) + return tmp_table, soup -def add_signatures(tmp_soup, tmp_dir, module_name, config): - """ Add signatures to the current soup and rewrite the html file""" +def get_anchor_list(anchor, soup): + a_list = [] + # go through all the links + for a in soup.find_all('a', href=True): + # find links with the same anchor + last_part_of_link = a['href'].rsplit('#', 1)[-1] + if last_part_of_link == anchor: + a_list.append(a) + return a_list - logging.debug(tmp_dir) - sign_counter = 0 - python_sign_counter = 0 - java_sign_counter = 0 +def append_python_signatures_to_table(soup, signatures, table): + description, soup = create_description(soup, "Python:", signatures) + description['class'] = 'python_language' + old = table.next_sibling + if old.name != 'table': + old = None + elif not 'python_language' in old.get('class', []): + old = None + # if already existed replace with the new + if old is None: + table.insert_after(description) + else: + old.replace_with(description) + table.insert_after(description) + return soup - if config.ADD_JAVA: - functions_file = "java_doc_txts/" + module_name + "/functions.txt" - if os.path.exists(functions_file): - with open(functions_file, 'r') as f: - java_signatures = f.read().split("\n") +def get_heading_text(a): + str = "" + element = a.parent.parent + if element is not None: + if element.has_attr('class'): + tmp_class = element["class"][0] + if "memitem:" in tmp_class and "python" not in tmp_class: + str = element.parent.find("tr").text + return str + +def append_python_signatures_to_heading(soup, signatures, element, href): + for s in signatures: + attrs = {'class': 'memitem:python'} + new_tr = soup.new_tag('tr', **attrs) + + attrs_left = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'} + new_td_left = soup.new_tag('td', **attrs_left) + new_td_left.append(str(s.get('ret', None))) + new_tr.append(new_td_left) + + attrs_right = {'class': 'memItemRight', 'valign': 'bottom'} + new_td_right = soup.new_tag('td', **attrs_right) + attrs_a = {'class': 'el', 'href': href} + new_a = soup.new_tag('a', **attrs_a) + new_a.append('cv2.' + str(s.get('name', None))) + new_td_right.append(new_a) + new_td_right.append("(" + s['arg'] +")") + + new_tr.append(new_td_right) + + old = element.next_sibling + if old is not None: + if old.name != 'tr': + old = None + elif not 'memitem:python' in old.get('class', []): + old = None + # if already existed replace with the new + if old is None: + element.insert_after(new_tr) else: - config.ADD_JAVA = False # This C++ module (module_name) may not exist in Java + old.replace_with(new_tr) + return soup - # the HTML tag & class being used to find functions - for function in tmp_soup.findAll("h2", {"class": "memtitle"}): - function_name = None - for c in function.contents: - if isinstance(c, bs4.element.NavigableString): - fn = str(c).encode("ascii","ignore").decode().strip() - if not fn.endswith('()'): # all functions have () in it's name - # enums, structures, etc - continue - function_name = fn[:-2] +def append_python_signature(function_variants, anchor_list, soup): + type = anchor_list[0].type + if type == "fn": + if len(anchor_list) == 1: + tmp_anchor = anchor_list[0].anchor + a_list = get_anchor_list(tmp_anchor, soup) + for a in a_list: + if a['href'] == "#" + tmp_anchor: + # Function Documentation (tables) + table = a.findNext('table') + if table is not None: + soup = append_python_signatures_to_table(soup, function_variants, table) + continue + str = get_heading_text(a) + if "Functions" in str: + soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href']) + print("One more") + return soup - if not function_name: - continue - - sign_counter += 1 - - cpp_table = function.findNext('table') - - if config.ADD_PYTHON: - signatures = config.python_signatures.get("cv::" + str(function_name), None) - if signatures: - print(function_name) - - description = create_description(tmp_soup, "Python:", signatures, function_name) - description['class'] = 'python_language' - old = cpp_table.next_sibling - if old.name != 'table': - old = None - elif not 'python_language' in old.get('class', []): - old = None - if old is None: - cpp_table.insert_after(description) - else: - old.replace_with(description) - python_sign_counter += 1 - - if config.ADD_JAVA: - for signature in java_signatures: - if function_name in signature: - create_description(cpp_table, tmp_soup, "Java:", signature, function_name) - java_sign_counter += 1 - break - - tmp_str = str(tmp_soup) +def update_html(file, soup): + tmp_str = str(soup) if os.name == 'nt': # if Windows - with open(tmp_dir, "wb") as tmp_file: + with open(file, "wb") as tmp_file: tmp_file.write(tmp_str.encode("ascii","ignore")) else: - with open(tmp_dir, "w") as tmp_file: + with open(file, "w") as tmp_file: tmp_file.write(tmp_str) - - logging.debug("Added [" + str(python_sign_counter) + \ - "/" + str(sign_counter) + "] Python signatures") - logging.debug("Added [" + str(java_sign_counter) + \ - "/" + str(sign_counter) + "] Java signatures") diff --git a/doc/tools/python_signatures.py b/doc/tools/python_signatures.py deleted file mode 100644 index 6908de278a..0000000000 --- a/doc/tools/python_signatures.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -This code adds Python signatures to the docs. - -TODO: -* clarify when there are several C++ signatures corresponding to a single Python function. - i.e: calcHist(): - http://docs.opencv.org/3.2.0/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d -* clarify special case: - http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 -""" -from __future__ import print_function -import os -import re -import sys -import logging - -loglevel=os.environ.get("LOGLEVEL", None) -if loglevel: - logging.basicConfig(level=loglevel) - -ADD_JAVA = False -ADD_PYTHON = True -ROOT_DIR = sys.argv[1] -PYTHON_SIGNATURES_FILE = sys.argv[2] - -import json -python_signatures = dict() -with open(PYTHON_SIGNATURES_FILE, "rt") as f: - python_signatures = json.load(f) - print("Loaded Python signatures: %d" % len(python_signatures)) - -class Configuration(): - def __init__(self): - self.ADD_PYTHON = ADD_PYTHON - self.python_signatures = python_signatures - self.ADD_JAVA = ADD_JAVA - -config = Configuration() - - -import html_functions - -soup = html_functions.load_html_file(ROOT_DIR + "index.html") -href_list = html_functions.get_links_list(soup, True) - -for link in href_list: - # add python signatures to the module - soup = html_functions.load_html_file(ROOT_DIR + link) - sub_href_list = html_functions.get_links_list(soup, True) - module_name = html_functions.get_text_between_substrings(link, "group__", ".html") - html_functions.add_signatures(soup, ROOT_DIR + link, module_name, config) - - # add python signatures to the sub-modules - link = re.sub(r"group__.+html", "", link) - for sub_link in sub_href_list: - tmp_dir = ROOT_DIR + link + sub_link - soup = html_functions.load_html_file(tmp_dir) - html_functions.add_signatures(soup, tmp_dir, module_name, config) From 164a77e7ad00b95b905a336354f6dd04f6653da7 Mon Sep 17 00:00:00 2001 From: tribta Date: Sun, 19 Nov 2017 16:22:00 -0500 Subject: [PATCH 4/6] add methods --- doc/tools/add_signatures.py | 4 +- doc/tools/html_functions.py | 138 +++++++++++++++++++++++------------- 2 files changed, 91 insertions(+), 51 deletions(-) diff --git a/doc/tools/add_signatures.py b/doc/tools/add_signatures.py index 05c029de25..95b57c2779 100644 --- a/doc/tools/add_signatures.py +++ b/doc/tools/add_signatures.py @@ -71,14 +71,14 @@ classes = root.findall("./compound[@kind='class']") for c in classes: c_name = c.find("./name").text name = ns_name + '::' + c_name - file = c.find("./filename").text + file = c.find("./filename").text #print('Class: {} => {}'.format(name, file)) files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict) # test for file in files_dict: soup = html_functions.load_html_file(ROOT_DIR + file) - if file == "d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html": + if file == "dd/d9e/classcv_1_1VideoWriter.html":#"d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html": anchor_list = files_dict[file] counter = 0 anchor_tmp_list = [] diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index ca3e096cde..9f9182a27b 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -26,15 +26,20 @@ def add_item(soup, new_row, is_parameter, text): new_row.append(new_item) return new_row, soup -def add_signature_to_table(soup, tmp_row, signature, language): +def add_signature_to_table(soup, tmp_row, signature, language, type): """ Add a signature to an html table""" new_item = soup.new_tag('td', style="padding-left: 0.5cm;") if str(signature.get('ret', None)) != "None": new_item.append(signature.get('ret') + ' =') - tmp_row.append(new_item) + tmp_row.append(new_item) - tmp_row, soup = add_item(soup, tmp_row, False, "cv2." + signature.get('name', None) + '(') + tmp_name = signature.get('name', None) + if type is not "method": + tmp_name = "cv2." + tmp_name + else: + tmp_name = "obj." + tmp_name + tmp_row, soup = add_item(soup, tmp_row, False, tmp_name + '(') tmp_row, soup = add_item(soup, tmp_row, True, signature['arg']) tmp_row, soup = add_item(soup, tmp_row, False, ')') return tmp_row, soup @@ -55,7 +60,7 @@ def add_bolded(soup, new_row, text): return new_row, soup -def create_description(soup, language, signatures): +def create_description(soup, language, signatures, type): """ Insert the new Python / Java table after the current html c++ table """ assert signatures tmp_table = soup.new_tag('table') @@ -64,7 +69,7 @@ def create_description(soup, language, signatures): new_row, soup = new_line(soup, tmp_table, new_row) for s in signatures: new_row, soup = new_line(soup, tmp_table, new_row) - new_row, soup = add_signature_to_table(soup, new_row, s, language) + new_row, soup = add_signature_to_table(soup, new_row, s, language, type) new_row, soup = new_line(soup, tmp_table, new_row) return tmp_table, soup @@ -79,25 +84,36 @@ def get_anchor_list(anchor, soup): a_list.append(a) return a_list -def append_python_signatures_to_table(soup, signatures, table): - description, soup = create_description(soup, "Python:", signatures) - description['class'] = 'python_language' - old = table.next_sibling - if old.name != 'table': - old = None - elif not 'python_language' in old.get('class', []): - old = None - # if already existed replace with the new - if old is None: - table.insert_after(description) +def is_static_method(element): + if element.name == "table": + tmp_element = element.find('td', {'class': 'memname'}) + if tmp_element is not None: + if 'static' in tmp_element.text: + return True else: - old.replace_with(description) - table.insert_after(description) + if element['class'][0] == 'memItemRight': + if "static" in element.previousSibling.text: + return True + return False + +def append_python_signatures_to_table(soup, signatures, table, type): + if type == "method": + if is_static_method(table): + type = "static" + type + description, soup = create_description(soup, "Python:", signatures, type) + description['class'] = 'python_language' + soup = insert_or_replace(soup, table, description, "table", "python_language") return soup def get_heading_text(a): str = "" - element = a.parent.parent + element = a.parent + if element is not None: + childs = element.find_all('a') + # the anchor should not be an argument of a function / method + if childs.index(a) is not 0: + return str + element = element.parent if element is not None: if element.has_attr('class'): tmp_class = element["class"][0] @@ -105,56 +121,80 @@ def get_heading_text(a): str = element.parent.find("tr").text return str -def append_python_signatures_to_heading(soup, signatures, element, href): +def insert_or_replace(soup, element, description, tag_name, tag_class): + old = element.next_sibling + if old is not None: + if old.name != tag_name: + old = None + elif not tag_class in old.get('class', []): + old = None + # if already existed replace with the new + if old is None: + element.insert_after(description) + else: + old.replace_with(description) + return soup + +def new_heading_td(soup, s, href, type): + if href is None: + attrs = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'} + new_td = soup.new_tag('td', **attrs) + new_td.append(str(s.get('ret', None))) + else: + attrs = {'class': 'memItemRight', 'valign': 'bottom'} + new_td = soup.new_tag('td', **attrs) + + # make the function name linkable + attrs_a = {'class': 'el', 'href': href} + new_a = soup.new_tag('a', **attrs_a) + tmp_name = str(s.get('name', None)) + if type is not "method": + tmp_name = "cv2." + tmp_name + else: + tmp_name = "obj." + tmp_name + new_a.append(tmp_name) + new_td.append(new_a) + + new_td.append("(" + s['arg'] +")") + return soup, new_td + +def append_python_signatures_to_heading(soup, signatures, element, href, type): + if type == "method": + if is_static_method(element): + type = "static" + type for s in signatures: attrs = {'class': 'memitem:python'} new_tr = soup.new_tag('tr', **attrs) - attrs_left = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'} - new_td_left = soup.new_tag('td', **attrs_left) - new_td_left.append(str(s.get('ret', None))) + soup, new_td_left = new_heading_td(soup, s, None, type) new_tr.append(new_td_left) - attrs_right = {'class': 'memItemRight', 'valign': 'bottom'} - new_td_right = soup.new_tag('td', **attrs_right) - attrs_a = {'class': 'el', 'href': href} - new_a = soup.new_tag('a', **attrs_a) - new_a.append('cv2.' + str(s.get('name', None))) - new_td_right.append(new_a) - new_td_right.append("(" + s['arg'] +")") - + soup, new_td_right = new_heading_td(soup, s, href, type) new_tr.append(new_td_right) - old = element.next_sibling - if old is not None: - if old.name != 'tr': - old = None - elif not 'memitem:python' in old.get('class', []): - old = None - # if already existed replace with the new - if old is None: - element.insert_after(new_tr) - else: - old.replace_with(new_tr) + soup = insert_or_replace(soup, element, new_tr, "tr", "memitem:python") return soup def append_python_signature(function_variants, anchor_list, soup): type = anchor_list[0].type - if type == "fn": + if type == "method" or type == "fn": if len(anchor_list) == 1: tmp_anchor = anchor_list[0].anchor a_list = get_anchor_list(tmp_anchor, soup) for a in a_list: if a['href'] == "#" + tmp_anchor: + tmp_element = a.parent + # ignore the More... link + if tmp_element is None or tmp_element['class'][0] == 'mdescRight': + continue # Function Documentation (tables) table = a.findNext('table') if table is not None: - soup = append_python_signatures_to_table(soup, function_variants, table) - continue - str = get_heading_text(a) - if "Functions" in str: - soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href']) - print("One more") + soup = append_python_signatures_to_table(soup, function_variants, table, type) + else: + str = get_heading_text(a) + if "Functions" in str: + soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href'], type) return soup def update_html(file, soup): From b15bc194ef6f5e8ae4c9a91ef7409082eb7c4bc5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 9 Dec 2017 04:38:50 +0000 Subject: [PATCH 5/6] doc: finalize Python signatures injection --- doc/CMakeLists.txt | 4 +- doc/tools/add_signatures.py | 94 ++++++------ doc/tools/doxygen_scan.py | 29 ++-- doc/tools/html_functions.py | 292 +++++++++++++----------------------- 4 files changed, 166 insertions(+), 253 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index d79b86c36b..f23a1b3047 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -218,7 +218,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) COMPONENT "docs" OPTIONAL ) - if(BUILD_opencv_python2) + if(PYTHON2_EXECUTABLE) add_custom_target(doxygen_python COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python2 @@ -226,7 +226,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) add_custom_target(doxygen DEPENDS doxygen_cpp doxygen_python ) - elseif(BUILD_opencv_python3) + elseif(PYTHON3_EXECUTABLE) add_custom_target(doxygen_python COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python3 diff --git a/doc/tools/add_signatures.py b/doc/tools/add_signatures.py index 95b57c2779..778dd33570 100644 --- a/doc/tools/add_signatures.py +++ b/doc/tools/add_signatures.py @@ -12,10 +12,15 @@ TODO: http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 """ from __future__ import print_function -import os -import re import sys +sys.dont_write_bytecode = True # Don't generate .pyc files / __pycache__ directories + +import os +from pprint import pprint +import re import logging +import json + import html_functions import doxygen_scan @@ -23,37 +28,23 @@ loglevel=os.environ.get("LOGLEVEL", None) if loglevel: logging.basicConfig(level=loglevel) - ROOT_DIR = sys.argv[1] PYTHON_SIGNATURES_FILE = sys.argv[2] -JAVA_PYTHON = sys.argv[3] +JAVA_OR_PYTHON = sys.argv[3] ADD_JAVA = False ADD_PYTHON = False -if JAVA_PYTHON == "python": +if JAVA_OR_PYTHON == "python": ADD_PYTHON = True -import json python_signatures = dict() with open(PYTHON_SIGNATURES_FILE, "rt") as f: python_signatures = json.load(f) print("Loaded Python signatures: %d" % len(python_signatures)) -# only name -> class -# name and ret -> constant -# name, ret, arg-> function / class method - -class Configuration(): - def __init__(self): - self.ADD_PYTHON = ADD_PYTHON - self.python_signatures = python_signatures - self.ADD_JAVA = ADD_JAVA - -config = Configuration() - import xml.etree.ElementTree as ET root = ET.parse(ROOT_DIR + 'opencv.tag') -files_dict = dict() +files_dict = {} # constants and function from opencv.tag namespaces = root.findall("./compound[@kind='namespace']") @@ -61,41 +52,48 @@ namespaces = root.findall("./compound[@kind='namespace']") for ns in namespaces: ns_name = ns.find("./name").text #print('NS: {}'.format(ns_name)) - - files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict) - files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict) + doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict) + doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict) # class methods from opencv.tag classes = root.findall("./compound[@kind='class']") #print("Found {} classes".format(len(classes))) for c in classes: c_name = c.find("./name").text - name = ns_name + '::' + c_name file = c.find("./filename").text - #print('Class: {} => {}'.format(name, file)) - files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict) + #print('Class: {} => {}'.format(c_name, file)) + doxygen_scan.scan_class_methods(c, c_name, files_dict) + +print('Doxygen files to scan: %s' % len(files_dict)) + +files_processed = 0 +files_skipped = 0 +symbols_processed = 0 -# test for file in files_dict: - soup = html_functions.load_html_file(ROOT_DIR + file) - if file == "dd/d9e/classcv_1_1VideoWriter.html":#"d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html": - anchor_list = files_dict[file] - counter = 0 - anchor_tmp_list = [] - for anchor in anchor_list: - counter += 1 - # if the next anchor shares the same C++ name (= same method/function), join them together - if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname: - anchor_tmp_list.append(anchor) - continue - else: - anchor_tmp_list.append(anchor) - # check if extists a python equivalent signature - for signature in python_signatures: # signature is a key with the C++ name - if signature == anchor.cppname: # if available name in python - # they should also have the same type - soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup) - #print(signature) - # reset anchor temporary list - anchor_tmp_list[:] = [] - html_functions.update_html(ROOT_DIR + file, soup) + #if file != "dd/d9e/classcv_1_1VideoWriter.html": + #if file != "d4/d86/group__imgproc__filter.html": + #if file != "df/dfb/group__imgproc__object.html": + # continue + #print('File: ' + file) + + anchor_list = files_dict[file] + active_anchors = [a for a in anchor_list if a.cppname in python_signatures] + if len(active_anchors) == 0: # no linked Python symbols + #print('Skip: ' + file) + files_skipped = files_skipped + 1 + continue + + active_anchors_dict = {a.anchor: a for a in active_anchors} + if len(active_anchors_dict) != len(active_anchors): + logging.info('Duplicate entries detected: %s -> %s (%s)' % (len(active_anchors), len(active_anchors_dict), file)) + + files_processed = files_processed + 1 + + #pprint(active_anchors) + symbols_processed = symbols_processed + len(active_anchors_dict) + + logging.info('File: %r' % file) + html_functions.insert_python_signatures(python_signatures, active_anchors_dict, ROOT_DIR + file) + +print('Done (processed files %d, symbols %d, skipped %d files)' % (files_processed, symbols_processed, files_skipped)) diff --git a/doc/tools/doxygen_scan.py b/doc/tools/doxygen_scan.py index 4567723302..a1ce60f618 100644 --- a/doc/tools/doxygen_scan.py +++ b/doc/tools/doxygen_scan.py @@ -1,20 +1,20 @@ -class Anchor(object): - anchor = "" - type = "" - cppname = "" +import traceback +class Symbol(object): def __init__(self, anchor, type, cppname): self.anchor = anchor self.type = type self.cppname = cppname + #if anchor == 'ga586ebfb0a7fb604b35a23d85391329be': + # print(repr(self)) + # traceback.print_stack() + + def __repr__(self): + return '%s:%s@%s' % (self.type, self.cppname, self.anchor) def add_to_file(files_dict, file, anchor): - if file in files_dict: - # if that file already exists as a key in the dictionary - files_dict[file].append(anchor) - else: - files_dict[file] = [anchor] - return files_dict + anchors = files_dict.setdefault(file, []) + anchors.append(anchor) def scan_namespace_constants(ns, ns_name, files_dict): @@ -25,8 +25,7 @@ def scan_namespace_constants(ns, ns_name, files_dict): file = c.find("./anchorfile").text anchor = c.find("./anchor").text #print(' CONST: {} => {}#{}'.format(name, file, anchor)) - files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name)) - return files_dict + add_to_file(files_dict, file, Symbol(anchor, "const", name)) def scan_namespace_functions(ns, ns_name, files_dict): functions = ns.findall("./member[@kind='function']") @@ -36,8 +35,7 @@ def scan_namespace_functions(ns, ns_name, files_dict): file = f.find("./anchorfile").text anchor = f.find("./anchor").text #print(' FN: {} => {}#{}'.format(name, file, anchor)) - files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name)) - return files_dict + add_to_file(files_dict, file, Symbol(anchor, "fn", name)) def scan_class_methods(c, c_name, files_dict): methods = c.findall("./member[@kind='function']") @@ -47,5 +45,4 @@ def scan_class_methods(c, c_name, files_dict): file = m.find("./anchorfile").text anchor = m.find("./anchor").text #print(' Method: {} => {}#{}'.format(name, file, anchor)) - files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name)) - return files_dict + add_to_file(files_dict, file, Symbol(anchor, "method", name)) diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index 9f9182a27b..c3fd3bd8e9 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -1,7 +1,10 @@ from __future__ import print_function +import sys + import logging import os from pprint import pprint +import traceback try: import bs4 @@ -13,195 +16,110 @@ except ImportError: def load_html_file(file_dir): """ Uses BeautifulSoup to load an html """ - with open(file_dir) as fp: + with open(file_dir, 'rb') as fp: soup = BeautifulSoup(fp, 'html.parser') return soup -def add_item(soup, new_row, is_parameter, text): - """ Adds a new html tag for the table with the signature """ - new_item = soup.new_tag('td') - if is_parameter: - new_item = soup.new_tag('td', **{'class': 'paramname'}) - new_item.append(text) - new_row.append(new_item) - return new_row, soup - -def add_signature_to_table(soup, tmp_row, signature, language, type): - """ Add a signature to an html table""" - new_item = soup.new_tag('td', style="padding-left: 0.5cm;") - - if str(signature.get('ret', None)) != "None": - new_item.append(signature.get('ret') + ' =') - tmp_row.append(new_item) - - tmp_name = signature.get('name', None) - if type is not "method": - tmp_name = "cv2." + tmp_name - else: - tmp_name = "obj." + tmp_name - tmp_row, soup = add_item(soup, tmp_row, False, tmp_name + '(') - tmp_row, soup = add_item(soup, tmp_row, True, signature['arg']) - tmp_row, soup = add_item(soup, tmp_row, False, ')') - return tmp_row, soup - - -def new_line(soup, tmp_table, new_row): - """ Adds a new line to the html table """ - tmp_table.append(new_row) - new_row = soup.new_tag('tr') - return new_row, soup - - -def add_bolded(soup, new_row, text): - """ Adds bolded text to the table """ - new_item = soup.new_tag('th', style="text-align:left") - new_item.append(text) - new_row.append(new_item) - return new_row, soup - - -def create_description(soup, language, signatures, type): - """ Insert the new Python / Java table after the current html c++ table """ - assert signatures - tmp_table = soup.new_tag('table') - new_row = soup.new_tag('tr') - new_row, soup = add_bolded(soup, new_row, language) - new_row, soup = new_line(soup, tmp_table, new_row) - for s in signatures: - new_row, soup = new_line(soup, tmp_table, new_row) - new_row, soup = add_signature_to_table(soup, new_row, s, language, type) - new_row, soup = new_line(soup, tmp_table, new_row) - return tmp_table, soup - - -def get_anchor_list(anchor, soup): - a_list = [] - # go through all the links - for a in soup.find_all('a', href=True): - # find links with the same anchor - last_part_of_link = a['href'].rsplit('#', 1)[-1] - if last_part_of_link == anchor: - a_list.append(a) - return a_list - -def is_static_method(element): - if element.name == "table": - tmp_element = element.find('td', {'class': 'memname'}) - if tmp_element is not None: - if 'static' in tmp_element.text: - return True - else: - if element['class'][0] == 'memItemRight': - if "static" in element.previousSibling.text: - return True - return False - -def append_python_signatures_to_table(soup, signatures, table, type): - if type == "method": - if is_static_method(table): - type = "static" + type - description, soup = create_description(soup, "Python:", signatures, type) - description['class'] = 'python_language' - soup = insert_or_replace(soup, table, description, "table", "python_language") - return soup - -def get_heading_text(a): - str = "" - element = a.parent - if element is not None: - childs = element.find_all('a') - # the anchor should not be an argument of a function / method - if childs.index(a) is not 0: - return str - element = element.parent - if element is not None: - if element.has_attr('class'): - tmp_class = element["class"][0] - if "memitem:" in tmp_class and "python" not in tmp_class: - str = element.parent.find("tr").text - return str - -def insert_or_replace(soup, element, description, tag_name, tag_class): - old = element.next_sibling - if old is not None: - if old.name != tag_name: - old = None - elif not tag_class in old.get('class', []): - old = None - # if already existed replace with the new - if old is None: - element.insert_after(description) - else: - old.replace_with(description) - return soup - -def new_heading_td(soup, s, href, type): - if href is None: - attrs = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'} - new_td = soup.new_tag('td', **attrs) - new_td.append(str(s.get('ret', None))) - else: - attrs = {'class': 'memItemRight', 'valign': 'bottom'} - new_td = soup.new_tag('td', **attrs) - - # make the function name linkable - attrs_a = {'class': 'el', 'href': href} - new_a = soup.new_tag('a', **attrs_a) - tmp_name = str(s.get('name', None)) - if type is not "method": - tmp_name = "cv2." + tmp_name - else: - tmp_name = "obj." + tmp_name - new_a.append(tmp_name) - new_td.append(new_a) - - new_td.append("(" + s['arg'] +")") - return soup, new_td - -def append_python_signatures_to_heading(soup, signatures, element, href, type): - if type == "method": - if is_static_method(element): - type = "static" + type - for s in signatures: - attrs = {'class': 'memitem:python'} - new_tr = soup.new_tag('tr', **attrs) - - soup, new_td_left = new_heading_td(soup, s, None, type) - new_tr.append(new_td_left) - - soup, new_td_right = new_heading_td(soup, s, href, type) - new_tr.append(new_td_right) - - soup = insert_or_replace(soup, element, new_tr, "tr", "memitem:python") - return soup - -def append_python_signature(function_variants, anchor_list, soup): - type = anchor_list[0].type - if type == "method" or type == "fn": - if len(anchor_list) == 1: - tmp_anchor = anchor_list[0].anchor - a_list = get_anchor_list(tmp_anchor, soup) - for a in a_list: - if a['href'] == "#" + tmp_anchor: - tmp_element = a.parent - # ignore the More... link - if tmp_element is None or tmp_element['class'][0] == 'mdescRight': - continue - # Function Documentation (tables) - table = a.findNext('table') - if table is not None: - soup = append_python_signatures_to_table(soup, function_variants, table, type) - else: - str = get_heading_text(a) - if "Functions" in str: - soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href'], type) - return soup - def update_html(file, soup): - tmp_str = str(soup) - if os.name == 'nt': # if Windows - with open(file, "wb") as tmp_file: - tmp_file.write(tmp_str.encode("ascii","ignore")) + s = str(soup) + if os.name == 'nt' or sys.version_info[0] == 3: # if Windows + s = s.encode('utf-8', 'ignore') + with open(file, 'wb') as f: + f.write(s) + + +def insert_python_signatures(python_signatures, symbols_dict, filepath): + soup = load_html_file(filepath) + entries = soup.find_all(lambda tag: tag.name == "a" and tag.has_attr('id')) + for e in entries: + anchor = e['id'] + if anchor in symbols_dict: + s = symbols_dict[anchor] + logging.info('Process: %r' % s) + if s.type == 'fn' or s.type == 'method': + process_fn(soup, e, python_signatures[s.cppname], s) + elif s.type == 'const': + process_const(soup, e, python_signatures[s.cppname], s) + else: + logging.error('unsupported type: %s' % s); + + update_html(filepath, soup) + + +def process_fn(soup, anchor, python_signature, symbol): + try: + r = anchor.find_next_sibling(class_='memitem').find(class_='memproto').find('table') + insert_python_fn_signature(soup, r, python_signature, symbol) + except: + logging.error("Can't process: %s" % symbol) + traceback.print_exc() + pprint(anchor) + + +def process_const(soup, anchor, python_signature, symbol): + try: + #pprint(anchor.parent) + description = append(soup.new_tag('div', **{'class' : ['python_language']}), + 'Python: ' + python_signature[0]['name']) + old = anchor.find_next_sibling('div', class_='python_language') + if old is None: + anchor.parent.append(description) + else: + old.replace_with(description) + #pprint(anchor.parent) + except: + logging.error("Can't process: %s" % symbol) + traceback.print_exc() + pprint(anchor) + + +def insert_python_fn_signature(soup, table, variants, symbol): + description = create_python_fn_description(soup, variants) + description['class'] = 'python_language' + soup = insert_or_replace(table, description, 'table', 'python_language') + return soup + + +def create_python_fn_description(soup, variants): + language = 'Python:' + table = soup.new_tag('table') + heading_row = soup.new_tag('th') + table.append( + append(soup.new_tag('tr'), + append(soup.new_tag('th', colspan=999, style="text-align:left"), language))) + for v in variants: + #logging.debug(v) + add_signature_to_table(soup, table, v, language, type) + #print(table) + return table + + +def add_signature_to_table(soup, table, signature, language, type): + """ Add a signature to an html table""" + row = soup.new_tag('tr') + row.append(soup.new_tag('td', style='width: 20px;')) + + if 'ret' in signature: + row.append(append(soup.new_tag('td'), signature['ret'])) + row.append(append(soup.new_tag('td'), '=')) else: - with open(file, "w") as tmp_file: - tmp_file.write(tmp_str) + row.append(soup.new_tag('td')) # return values + row.append(soup.new_tag('td')) # '=' + + row.append(append(soup.new_tag('td'), signature['name'] + '(')) + row.append(append(soup.new_tag('td', **{'class': 'paramname'}), signature['arg'])) + row.append(append(soup.new_tag('td'), ')')) + table.append(row) + + +def append(target, obj): + target.append(obj) + return target + + +def insert_or_replace(element_before, new_element, tag, tag_class): + old = element_before.find_next_sibling(tag, class_=tag_class) + if old is None: + element_before.insert_after(new_element) + else: + old.replace_with(new_element) From 768f4cb7bc18872c3a8179e79a2e9c8406ed7c1f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 9 Dec 2017 18:54:52 +0000 Subject: [PATCH 6/6] python: 'sub-module' for binding sources and documentation meta information --- cmake/OpenCVModule.cmake | 2 +- doc/CMakeLists.txt | 39 ++++++----- modules/python/CMakeLists.txt | 2 + modules/python/bindings/CMakeLists.txt | 69 ++++++++++++++++++++ modules/python/common.cmake | 90 +++++--------------------- modules/python/src2/cv2.cpp | 6 ++ modules/python/src2/gen2.py | 15 +---- 7 files changed, 118 insertions(+), 105 deletions(-) create mode 100644 modules/python/bindings/CMakeLists.txt diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index 4c1eaf650c..17f5b3b7dd 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -104,7 +104,7 @@ macro(ocv_add_dependencies full_modname) list(FIND OPENCV_MODULE_${full_modname}_WRAPPERS "python" __python_idx) if (NOT __python_idx EQUAL -1) list(REMOVE_ITEM OPENCV_MODULE_${full_modname}_WRAPPERS "python") - list(APPEND OPENCV_MODULE_${full_modname}_WRAPPERS "python2" "python3") + list(APPEND OPENCV_MODULE_${full_modname}_WRAPPERS "python_bindings_generator" "python2" "python3") endif() unset(__python_idx) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index f23a1b3047..aa049f9189 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -33,7 +33,7 @@ endif(HAVE_DOC_GENERATOR) if(BUILD_DOCS AND DOXYGEN_FOUND) # not documented modules list - list(APPEND blacklist "ts" "java" "python2" "python3" "js" "world") + list(APPEND blacklist "ts" "java" "python_bindings_generator" "python2" "python3" "js" "world") unset(CMAKE_DOXYGEN_TUTORIAL_CONTRIB_ROOT) unset(CMAKE_DOXYGEN_TUTORIAL_JS_ROOT) @@ -104,7 +104,6 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) list(APPEND deps ${bib_file}) endif() # Reference entry - # set(one_ref "@ref ${m} | ${m}\n") set(one_ref "\t- ${m}. @ref ${m}\n") list(FIND EXTRA_MODULES ${m} _pos) if(${_pos} EQUAL -1) @@ -205,9 +204,8 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) list(APPEND js_tutorials_assets_deps "${f}" "${opencv_tutorial_html_dir}/${fname}") endforeach() - set(doxygen_result "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/modules.html") - - add_custom_target(doxygen_cpp + add_custom_target( + doxygen_cpp COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} DEPENDS ${doxyfile} ${rootfile} ${bibfile} ${deps} ${js_tutorials_assets_deps} COMMENT "Generate Doxygen documentation" @@ -218,18 +216,27 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) COMPONENT "docs" OPTIONAL ) - if(PYTHON2_EXECUTABLE) + if(NOT DEFINED HAVE_PYTHON_BS4 AND PYTHON_DEFAULT_EXECUTABLE) + # Documentation post-processing tool requires BuautifulSoup Python package + execute_process(COMMAND "${PYTHON_DEFAULT_EXECUTABLE}" -c "import bs4; from bs4 import BeautifulSoup; print(bs4.__version__)" + RESULT_VARIABLE _result + OUTPUT_VARIABLE _bs4_version + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(NOT _result EQUAL 0) + set(HAVE_PYTHON_BS4 0 CACHE INTERNAL "") + else() + message(STATUS "Python BeautifulSoup (bs4) version: ${_bs4_version}") + set(HAVE_PYTHON_BS4 1 CACHE INTERNAL "") + endif() + endif() + + if(PYTHON_DEFAULT_EXECUTABLE AND HAVE_PYTHON_BS4 + AND OPENCV_PYTHON_SIGNATURES_FILE AND TARGET gen_opencv_python_source) add_custom_target(doxygen_python - COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python" - DEPENDS "${doxygen_result}" gen_opencv_python2 - ) - add_custom_target(doxygen - DEPENDS doxygen_cpp doxygen_python - ) - elseif(PYTHON3_EXECUTABLE) - add_custom_target(doxygen_python - COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python" - DEPENDS "${doxygen_result}" gen_opencv_python3 + COMMAND ${PYTHON_DEFAULT_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON_SIGNATURES_FILE}" "python" + DEPENDS doxygen_cpp gen_opencv_python_source + COMMENT "Inject Python signatures into documentation" ) add_custom_target(doxygen DEPENDS doxygen_cpp doxygen_python diff --git a/modules/python/CMakeLists.txt b/modules/python/CMakeLists.txt index d85a516791..dffe9a794e 100644 --- a/modules/python/CMakeLists.txt +++ b/modules/python/CMakeLists.txt @@ -2,6 +2,8 @@ # CMake file for python support # ---------------------------------------------------------------------------- +add_subdirectory(bindings) + if(ANDROID OR APPLE_FRAMEWORK OR WINRT) set(__disable_python2 ON) set(__disable_python3 ON) diff --git a/modules/python/bindings/CMakeLists.txt b/modules/python/bindings/CMakeLists.txt new file mode 100644 index 0000000000..e407ea9a11 --- /dev/null +++ b/modules/python/bindings/CMakeLists.txt @@ -0,0 +1,69 @@ +set(MODULE_NAME "python_bindings_generator") +ocv_add_module(${MODULE_NAME} INTERNAL) + +set(OPENCV_PYTHON_SIGNATURES_FILE "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_signatures.json" CACHE INTERNAL "") +set(OPENCV_PYTHON_BINDINGS_DIR "${CMAKE_CURRENT_BINARY_DIR}" CACHE INTERNAL "") + +# This file is included from a subdirectory +set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") + +# get list of modules to wrap +set(OPENCV_PYTHON_MODULES) +foreach(m ${OPENCV_MODULES_BUILD}) + if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";${MODULE_NAME};" AND HAVE_${m}) + list(APPEND OPENCV_PYTHON_MODULES ${m}) + #message(STATUS "\t${m}") + endif() +endforeach() + +set(opencv_hdrs "") +set(opencv_userdef_hdrs "") +foreach(m ${OPENCV_PYTHON_MODULES}) + list(APPEND opencv_hdrs ${OPENCV_MODULE_${m}_HEADERS}) + file(GLOB userdef_hdrs ${OPENCV_MODULE_${m}_LOCATION}/misc/python/pyopencv*.hpp) + list(APPEND opencv_userdef_hdrs ${userdef_hdrs}) +endforeach(m) + +# header blacklist +ocv_list_filterout(opencv_hdrs "modules/.*\\\\.h$") +ocv_list_filterout(opencv_hdrs "modules/core/.*/cuda") +ocv_list_filterout(opencv_hdrs "modules/cuda.*") +ocv_list_filterout(opencv_hdrs "modules/cudev") +ocv_list_filterout(opencv_hdrs "modules/core/.*/hal/") +ocv_list_filterout(opencv_hdrs "modules/.+/utils/.*") +ocv_list_filterout(opencv_hdrs "modules/.*\\\\.inl\\\\.h*") +ocv_list_filterout(opencv_hdrs "modules/.*_inl\\\\.h*") +ocv_list_filterout(opencv_hdrs "modules/.*\\\\.details\\\\.h*") +ocv_list_filterout(opencv_hdrs "modules/.*/detection_based_tracker\\\\.hpp") # Conditional compilation + +set(cv2_generated_hdrs + "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_include.h" + "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_funcs.h" + "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_types.h" + "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_type_reg.h" + "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_ns_reg.h" +) + +set(cv2_generated_files ${cv2_generated_hdrs} + "${OPENCV_PYTHON_SIGNATURES_FILE}" +) + +string(REPLACE ";" "\n" opencv_hdrs_ "${opencv_hdrs}") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${opencv_hdrs_}") +add_custom_command( + OUTPUT ${cv2_generated_files} + COMMAND ${PYTHON_DEFAULT_EXECUTABLE} "${PYTHON_SOURCE_DIR}/src2/gen2.py" ${CMAKE_CURRENT_BINARY_DIR} "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" + DEPENDS ${PYTHON_SOURCE_DIR}/src2/gen2.py + DEPENDS ${PYTHON_SOURCE_DIR}/src2/hdr_parser.py + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/headers.txt + DEPENDS ${opencv_hdrs} + COMMENT "Generate files for Python bindings and documentation" +) + +add_custom_target(gen_opencv_python_source DEPENDS ${cv2_generated_files}) + +set(cv2_custom_hdr "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_custom_headers.h") +file(WRITE ${cv2_custom_hdr} "//user-defined headers\n") +foreach(uh ${opencv_userdef_hdrs}) + file(APPEND ${cv2_custom_hdr} "#include \"${uh}\"\n") +endforeach(uh) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 9f87a043a3..fcccffe2ed 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,6 +1,17 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") +ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator) + +ocv_module_include_directories( + "${${PYTHON}_INCLUDE_PATH}" +) +include_directories( + ${${PYTHON}_NUMPY_INCLUDE_DIRS} + "${PYTHON_SOURCE_DIR}/src2" + "${OPENCV_PYTHON_BINDINGS_DIR}" +) + # try to use dynamic symbols linking with libpython.so set(OPENCV_FORCE_PYTHON_LIBS OFF CACHE BOOL "") string(REPLACE "-Wl,--no-undefined" "" CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS}") @@ -8,80 +19,8 @@ if(NOT WIN32 AND NOT APPLE AND NOT OPENCV_PYTHON_SKIP_LINKER_EXCLUDE_LIBS) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--exclude-libs=ALL") endif() -ocv_add_module(${MODULE_NAME} BINDINGS) - -ocv_module_include_directories( - "${${PYTHON}_INCLUDE_PATH}" - ${${PYTHON}_NUMPY_INCLUDE_DIRS} - "${PYTHON_SOURCE_DIR}/src2" - ) - -# get list of modules to wrap -# message(STATUS "Wrapped in ${MODULE_NAME}:") -set(OPENCV_PYTHON_MODULES) -foreach(m ${OPENCV_MODULES_BUILD}) - if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";${MODULE_NAME};" AND HAVE_${m}) - list(APPEND OPENCV_PYTHON_MODULES ${m}) - # message(STATUS "\t${m}") - endif() -endforeach() - -set(opencv_hdrs "") -set(opencv_userdef_hdrs "") -foreach(m ${OPENCV_PYTHON_MODULES}) - list(APPEND opencv_hdrs ${OPENCV_MODULE_${m}_HEADERS}) - file(GLOB userdef_hdrs ${OPENCV_MODULE_${m}_LOCATION}/misc/python/pyopencv*.hpp) - list(APPEND opencv_userdef_hdrs ${userdef_hdrs}) -endforeach(m) - -# header blacklist -ocv_list_filterout(opencv_hdrs "modules/.*\\\\.h$") -ocv_list_filterout(opencv_hdrs "modules/core/.*/cuda") -ocv_list_filterout(opencv_hdrs "modules/cuda.*") -ocv_list_filterout(opencv_hdrs "modules/cudev") -ocv_list_filterout(opencv_hdrs "modules/core/.*/hal/") -ocv_list_filterout(opencv_hdrs "modules/.+/utils/.*") -ocv_list_filterout(opencv_hdrs "modules/.*\\\\.inl\\\\.h*") -ocv_list_filterout(opencv_hdrs "modules/.*_inl\\\\.h*") -ocv_list_filterout(opencv_hdrs "modules/.*\\\\.details\\\\.h*") -ocv_list_filterout(opencv_hdrs "modules/.*/detection_based_tracker\\\\.hpp") # Conditional compilation - -set(cv2_generated_hdrs - "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_include.h" - "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_funcs.h" - "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_types.h" - "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_type_reg.h" - "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_generated_ns_reg.h" -) - -set(OPENCV_${PYTHON}_SIGNATURES_FILE "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_signatures.json" CACHE INTERNAL "") - -set(cv2_generated_files ${cv2_generated_hdrs} - "${OPENCV_${PYTHON}_SIGNATURES_FILE}" -) - -string(REPLACE ";" "\n" opencv_hdrs_ "${opencv_hdrs}") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${opencv_hdrs_}") -add_custom_command( - OUTPUT ${cv2_generated_files} - COMMAND ${PYTHON_DEFAULT_EXECUTABLE} "${PYTHON_SOURCE_DIR}/src2/gen2.py" ${CMAKE_CURRENT_BINARY_DIR} "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${PYTHON}" - DEPENDS ${PYTHON_SOURCE_DIR}/src2/gen2.py - DEPENDS ${PYTHON_SOURCE_DIR}/src2/hdr_parser.py - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/headers.txt - DEPENDS ${opencv_hdrs} - COMMENT "Generate files for ${the_module}" -) - -add_custom_target(gen_${the_module} DEPENDS ${cv2_generated_files}) - -set(cv2_custom_hdr "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_custom_headers.h") -file(WRITE ${cv2_custom_hdr} "//user-defined headers\n") -foreach(uh ${opencv_userdef_hdrs}) - file(APPEND ${cv2_custom_hdr} "#include \"${uh}\"\n") -endforeach(uh) - ocv_add_library(${the_module} MODULE ${PYTHON_SOURCE_DIR}/src2/cv2.cpp ${cv2_generated_hdrs} ${opencv_userdef_hdrs} ${cv2_custom_hdr}) -add_dependencies(${the_module} gen_${the_module}) +add_dependencies(${the_module} gen_opencv_python_source) if(APPLE) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") @@ -92,7 +31,10 @@ elseif(WIN32 OR OPENCV_FORCE_PYTHON_LIBS) ocv_target_link_libraries(${the_module} LINK_PRIVATE ${${PYTHON}_LIBRARIES}) endif() endif() -ocv_target_link_libraries(${the_module} LINK_PRIVATE ${OPENCV_MODULE_${the_module}_DEPS}) + +set(deps ${OPENCV_MODULE_${the_module}_DEPS}) +list(REMOVE_ITEM deps opencv_python_bindings_generator) # don't add dummy module to target_link_libraries list +ocv_target_link_libraries(${the_module} LINK_PRIVATE ${deps}) if(DEFINED ${PYTHON}_CVPY_SUFFIX) set(CVPY_SUFFIX "${${PYTHON}_CVPY_SUFFIX}") diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 30c84f46b9..8f1eb3a214 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -9,6 +9,12 @@ #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include +#if PY_MAJOR_VERSION >= 3 +# define CV_PYTHON_TYPE_HEAD_INIT() PyVarObject_HEAD_INIT(&PyType_Type, 0) +#else +# define CV_PYTHON_TYPE_HEAD_INIT() PyObject_HEAD_INIT(&PyType_Type) 0, +#endif + #include "pyopencv_generated_include.h" #include "opencv2/core/types_c.h" diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 4f15715ccc..27928bc485 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -46,20 +46,7 @@ gen_template_func_body = Template("""$code_decl } """) -py_major_version = sys.version_info[0] -if __name__ == "__main__": - if len(sys.argv) > 3: - if sys.argv[3] == 'PYTHON3': - py_major_version = 3 - elif sys.argv[3] == 'PYTHON2': - py_major_version = 2 - else: - raise Exception('Incorrect argument: expected PYTHON2 or PYTHON3, received: ' + sys.argv[3]) -if py_major_version >= 3: - head_init_str = "PyVarObject_HEAD_INIT(&PyType_Type, 0)" -else: - head_init_str = """PyObject_HEAD_INIT(&PyType_Type) -0,""" +head_init_str = "CV_PYTHON_TYPE_HEAD_INIT()" gen_template_simple_type_decl = Template(""" struct pyopencv_${name}_t