From 20fd6393e4e2acc408d0a66af9d4b65e66a061f5 Mon Sep 17 00:00:00 2001 From: pycook Date: Wed, 28 Aug 2019 20:34:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=87=E7=BA=A7=E5=90=8E=E7=AB=AF=E5=B9=B6?= =?UTF-8?q?=E5=BC=80=E6=BA=90UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 7 + .gitignore | 104 +++- LICENSE | 353 +------------- Makefile | 25 + Pipfile | 59 +++ README.md | 56 ++- __init__.py | 120 ----- api/__init__.py | 1 + api/app.py | 166 +++++++ {lib => api/commands}/__init__.py | 3 - api/commands/common.py | 152 ++++++ api/extensions.py | 21 + api/flask_cas/__init__.py | 78 +++ api/flask_cas/cas_urls.py | 122 +++++ api/flask_cas/routing.py | 164 +++++++ api/lib/__init__.py | 1 + api/lib/cmdb/__init__.py | 1 + api/lib/cmdb/attribute.py | 150 ++++++ api/lib/cmdb/cache.py | 138 ++++++ api/lib/cmdb/ci.py | 552 ++++++++++++++++++++++ api/lib/cmdb/ci_type.py | 392 ++++++++++++++++ api/lib/cmdb/const.py | 148 ++++++ api/lib/cmdb/history.py | 126 +++++ api/lib/cmdb/preference.py | 137 ++++++ api/lib/cmdb/query_sql.py | 63 +++ api/lib/cmdb/relation_type.py | 37 ++ api/lib/cmdb/search.py | 356 ++++++++++++++ api/lib/cmdb/value.py | 138 ++++++ api/lib/database.py | 121 +++++ api/lib/decorator.py | 35 ++ api/lib/exception.py | 5 + api/lib/http_cli.py | 48 ++ api/lib/mail.py | 49 ++ api/lib/perm/__init__.py | 1 + api/lib/perm/acl.py | 139 ++++++ api/lib/perm/auth.py | 102 ++++ {lib => api/lib}/utils.py | 77 ++- api/models/__init__.py | 5 + {models => api/models}/account.py | 112 ++--- api/models/cmdb.py | 325 +++++++++++++ api/resource.py | 45 ++ api/settings.py.example | 77 +++ api/tasks/__init__.py | 1 + {tasks => api/tasks}/cmdb.py | 19 +- api/tasks/test.py | 9 + api/views/__init__.py | 31 ++ api/views/account.py | 47 ++ api/views/cmdb/__init__.py | 1 + api/views/cmdb/attribute.py | 65 +++ api/views/cmdb/ci.py | 218 +++++++++ api/views/cmdb/ci_relation.py | 66 +++ api/views/cmdb/ci_type.py | 202 ++++++++ api/views/cmdb/ci_type_relation.py | 49 ++ api/views/cmdb/history.py | 63 +++ api/views/cmdb/preference.py | 89 ++++ api/views/cmdb/relation_type.py | 37 ++ api/views/permission.py | 49 ++ autoapp.py | 14 + celery_worker.py | 9 + cmdb_api.md | 11 +- command/__init__.py | 1 - config-sample.cfg | 61 --- core/__init__.py | 11 - core/account.py | 98 ---- core/attribute.py | 152 ------ core/ci.py | 216 --------- core/ci_relation.py | 70 --- core/ci_type.py | 89 ---- core/ci_type_relation.py | 55 --- core/history.py | 116 ----- core/statis.py | 12 - extensions.py | 19 - gunicornserver.py | 72 --- lib/account.py | 145 ------ lib/attribute.py | 168 ------- lib/auth.py | 64 --- lib/ci.py | 704 ---------------------------- lib/ci_type.py | 316 ------------- lib/const.py | 101 ---- lib/decorator.py | 74 --- lib/exception.py | 17 - lib/history.py | 75 --- lib/query_sql.py | 107 ----- lib/search.py | 365 -------------- lib/template/__init__.py | 1 - lib/template/filters.py | 9 - lib/value.py | 170 ------- manage.py | 77 --- models/__init__.py | 13 - models/attribute.py | 86 ---- models/ci.py | 19 - models/ci_relation.py | 26 - models/ci_type.py | 129 ----- models/ci_type_relation.py | 26 - models/ci_value.py | 117 ----- models/history.py | 50 -- models/statis.py | 20 - permissions.py | 9 - requirements/default.txt | 14 - setup.cfg | 11 + tasks/__init__.py | 1 - tasks/statis.py | 21 - templates/search.xml | 27 -- templates/search_tidy.xml | 19 - tests/__init__.py | 1 + tests/conftest.py | 24 + tests/test_cmdb_attribute.py | 1 + tests/test_cmdb_ci.py | 10 + tests/test_cmdb_ci_realtion.py | 1 + tests/test_cmdb_ci_type.py | 1 + tests/test_cmdb_ci_type_relation.py | 1 + tests/test_cmdb_history.py | 1 + tests/test_cmdb_preference.py | 1 + tests/test_cmdb_relation_type.py | 1 + 114 files changed, 5243 insertions(+), 4543 deletions(-) create mode 100644 .env create mode 100644 Makefile create mode 100644 Pipfile delete mode 100644 __init__.py create mode 100644 api/__init__.py create mode 100644 api/app.py rename {lib => api/commands}/__init__.py (63%) create mode 100644 api/commands/common.py create mode 100644 api/extensions.py create mode 100644 api/flask_cas/__init__.py create mode 100644 api/flask_cas/cas_urls.py create mode 100644 api/flask_cas/routing.py create mode 100644 api/lib/__init__.py create mode 100644 api/lib/cmdb/__init__.py create mode 100644 api/lib/cmdb/attribute.py create mode 100644 api/lib/cmdb/cache.py create mode 100644 api/lib/cmdb/ci.py create mode 100644 api/lib/cmdb/ci_type.py create mode 100644 api/lib/cmdb/const.py create mode 100644 api/lib/cmdb/history.py create mode 100644 api/lib/cmdb/preference.py create mode 100644 api/lib/cmdb/query_sql.py create mode 100644 api/lib/cmdb/relation_type.py create mode 100644 api/lib/cmdb/search.py create mode 100644 api/lib/cmdb/value.py create mode 100644 api/lib/database.py create mode 100644 api/lib/decorator.py create mode 100644 api/lib/exception.py create mode 100644 api/lib/http_cli.py create mode 100644 api/lib/mail.py create mode 100644 api/lib/perm/__init__.py create mode 100644 api/lib/perm/acl.py create mode 100644 api/lib/perm/auth.py rename {lib => api/lib}/utils.py (53%) create mode 100644 api/models/__init__.py rename {models => api/models}/account.py (69%) create mode 100644 api/models/cmdb.py create mode 100644 api/resource.py create mode 100644 api/settings.py.example create mode 100644 api/tasks/__init__.py rename {tasks => api/tasks}/cmdb.py (51%) create mode 100644 api/tasks/test.py create mode 100644 api/views/__init__.py create mode 100644 api/views/account.py create mode 100644 api/views/cmdb/__init__.py create mode 100644 api/views/cmdb/attribute.py create mode 100644 api/views/cmdb/ci.py create mode 100644 api/views/cmdb/ci_relation.py create mode 100644 api/views/cmdb/ci_type.py create mode 100644 api/views/cmdb/ci_type_relation.py create mode 100644 api/views/cmdb/history.py create mode 100644 api/views/cmdb/preference.py create mode 100644 api/views/cmdb/relation_type.py create mode 100644 api/views/permission.py create mode 100644 autoapp.py create mode 100644 celery_worker.py delete mode 100644 command/__init__.py delete mode 100644 config-sample.cfg delete mode 100644 core/__init__.py delete mode 100644 core/account.py delete mode 100644 core/attribute.py delete mode 100644 core/ci.py delete mode 100644 core/ci_relation.py delete mode 100644 core/ci_type.py delete mode 100644 core/ci_type_relation.py delete mode 100644 core/history.py delete mode 100644 core/statis.py delete mode 100644 extensions.py delete mode 100644 gunicornserver.py delete mode 100644 lib/account.py delete mode 100644 lib/attribute.py delete mode 100644 lib/auth.py delete mode 100644 lib/ci.py delete mode 100644 lib/ci_type.py delete mode 100644 lib/const.py delete mode 100644 lib/decorator.py delete mode 100644 lib/exception.py delete mode 100644 lib/history.py delete mode 100644 lib/query_sql.py delete mode 100644 lib/search.py delete mode 100644 lib/template/__init__.py delete mode 100644 lib/template/filters.py delete mode 100644 lib/value.py delete mode 100644 manage.py delete mode 100644 models/__init__.py delete mode 100644 models/attribute.py delete mode 100644 models/ci.py delete mode 100644 models/ci_relation.py delete mode 100644 models/ci_type.py delete mode 100644 models/ci_type_relation.py delete mode 100644 models/ci_value.py delete mode 100644 models/history.py delete mode 100644 models/statis.py delete mode 100644 permissions.py delete mode 100644 requirements/default.txt create mode 100644 setup.cfg delete mode 100644 tasks/__init__.py delete mode 100644 tasks/statis.py delete mode 100644 templates/search.xml delete mode 100644 templates/search_tidy.xml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_cmdb_attribute.py create mode 100644 tests/test_cmdb_ci.py create mode 100644 tests/test_cmdb_ci_realtion.py create mode 100644 tests/test_cmdb_ci_type.py create mode 100644 tests/test_cmdb_ci_type_relation.py create mode 100644 tests/test_cmdb_history.py create mode 100644 tests/test_cmdb_preference.py create mode 100644 tests/test_cmdb_relation_type.py diff --git a/.env b/.env new file mode 100644 index 0000000..c005fda --- /dev/null +++ b/.env @@ -0,0 +1,7 @@ +# Environment variable overrides for local development +FLASK_APP=autoapp.py +FLASK_DEBUG=1 +FLASK_ENV=development +GUNICORN_WORKERS=1 +LOG_LEVEL=debug +SECRET_KEY='YourSecretKey' diff --git a/.gitignore b/.gitignore index f76ea86..e07e230 100755 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,95 @@ *~ -*.pyc .idea -data -logs/* +.vscode +migrates +*/logs/* +config.cfg *.sql -test/* -tools/* -cmdb_agent/* +logs/* *.log +*_packed.js +*_packed.css *.orig *.zip -*.swp -config.cfg -*.tar.gz -core/special.py -lib/special -lib/audit* -templates/*audit* -codeLin* -lib/spec_* +nohup.out +.DS_Store +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +#lib +#lib64 +Pipfile.lock + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.pytest_cache + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +docs/_build + +# Virtualenvs +env/ + + +# Configuration +api/settings.py +tests/settings.py + +# Development database +*.db + +# UI +ui/.DS_Store +ui/node_modules +ui/dist + +# local env files +ui/.env.local +ui/.env.*.local + +# Log files +ui/npm-debug.log* +ui/yarn-debug.log* +ui/yarn-error.log* +ui/yarn.lock + +# Editor directories and files +ui/.idea +ui/.vscode +ui/*.suo +ui/*.ntvs* +ui/*.njsproj +ui/*.sln +ui/*.sw* + + diff --git a/LICENSE b/LICENSE index 8cdb845..3098ec8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,340 +1,21 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 +MIT License - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. +Copyright (c) pycook - Preamble +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - {description} - Copyright (C) {year} {fullname} - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - {signature of Ty Coon}, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cbd4424 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +.PHONY: docs test + +help: + @echo " env create a development environment using virtualenv" + @echo " deps install dependencies using pip" + @echo " clean remove unwanted files like .pyc's" + @echo " lint check style with flake8" + @echo " test run all your tests using py.test" + +env: + sudo easy_install pip && \ + pip install pipenv && + make deps + +deps: + pipenv install --dev + +clean: + python manage.py clean + +lint: + flake8 --exclude=env . + +test: +py.test tests \ No newline at end of file diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..7459fd8 --- /dev/null +++ b/Pipfile @@ -0,0 +1,59 @@ +[[source]] +url = "https://mirrors.aliyun.com/pypi/simple" +verify_ssl = true +name = "pypi" + +[packages] +# Flask +Flask = "==1.0.3" +Werkzeug = "==0.15.4" +click = ">=5.0" +# Api +Flask-RESTful = "==0.3.7" +# Database +Flask-SQLAlchemy = "==2.4.0" +SQLAlchemy = "==1.3.5" +PyMySQL = "==0.9.3" +redis = "==3.2.1" +# Migrations +Flask-Migrate = "==2.5.2" +# Deployment +gevent = "==1.4.0" +gunicorn = ">=19.1.1" +supervisor = "==4.0.3" +# Auth +Flask-Login = "==0.4.1" +Flask-Bcrypt = "==0.7.1" +Flask-Cors = ">=3.0.8" +# Caching +Flask-Caching = ">=1.0.0" +# Environment variable parsing +environs = "==4.2.0" +marshmallow = "==2.20.2" +# async tasks +celery = "==4.3.0" +more-itertools = "==5.0.0" +kombu = "==4.4.0" +# other +six = "==1.12.0" +bs4 = ">=0.0.1" +toposort = ">=1.5" +requests = ">=2.22.0" +PyJWT = ">=1.7.1" + +[dev-packages] +# Testing +pytest = "==4.6.5" +WebTest = "==2.0.33" +factory-boy = "==2.12.*" +pdbpp = "==0.10.0" +# Lint and code style +flake8 = "==3.7.7" +flake8-blind-except = "==0.1.1" +flake8-debugger = "==3.1.0" +flake8-docstrings = "==1.3.0" +flake8-isort = "==2.7.0" +isort = "==4.3.21" +pep8-naming = "==0.8.2" +pydocstyle = "==3.0.0" + diff --git a/README.md b/README.md index ced09e7..61a4dbb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,54 @@ -## cmdb +

CMDB

+
+尽可能实现比较通用的运维资产数据的配置和管理 +
+ +
+ +[![License](https://img.shields.io/badge/License-mit-brightgreen)](https://github.com/pycook/cmdb/blob/master/LICENSE) +[![UI](https://img.shields.io/badge/UI-Ant%20Design%20Pro%20Vue-brightgreen)](https://github.com/sendya/ant-design-pro-vue) +[![API](https://img.shields.io/badge/API-Flask-brightgreen)](https://github.com/pallets/flask) + +
+ + + +- 在线预览: [CMDB](url "http://39.100.252.148:8000") + - username: admin + - password: admin + +Overview +---- +![基础资源视图](ui/public/cmdb01.jpeg) + +![模型配置](ui/public/cmdb02.jpeg) + +环境和依赖 +---- +- 存储: mysql, redis +- python版本: python2.7, >=python3.6 + + +安装 +---- +- 创建数据库cmdb + +- 拉取代码 +```bash +git clone https://github.com/pycook/cmdb.git +cd cmdb +cp api/settings.py.example api/settings.py +``` +设置api/settings.py里的database + +- 安装库 + - 后端: ```pipenv run pipenv install``` + - 前端: ```cd ui && yarn install && cd ..``` + +- 创建数据库表 ```flask run flask db-setup``` + +- 启动服务 + - 后端: ```pipenv run flask run``` + - 前端: ```cd ui && yarn run serve``` + - 浏览器打开: [http://127.0.0.1:8000](http://127.0.0.1:8000) -### cmdb即配置管理数据库 -### 该部分为API，Portal即将单独开源 \ No newline at end of file diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 94f505e..0000000 --- a/__init__.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import logging -from logging.handlers import SMTPHandler -from logging.handlers import TimedRotatingFileHandler - -from flask import Flask -from flask import request -from flask import g -from flask.ext.babel import Babel -from flask.ext.principal import identity_loaded -from flask.ext.principal import Principal - -import core -from extensions import db -from extensions import mail -from extensions import cache -from extensions import celery -from extensions import rd -from models.account import User -from lib.template import filters - - -APP_NAME = "CMDB-API" - -MODULES = ( - (core.attribute, "/api/v0.1/attributes"), - (core.citype, "/api/v0.1/citypes"), - (core.cityperelation, "/api/v0.1/cityperelations"), - (core.cirelation, "/api/v0.1/cirelations"), - (core.ci, "/api/v0.1/ci"), - (core.history, "/api/v0.1/history"), - (core.account, "/api/v0.1/accounts"), - (core.special, ""), -) - - -def make_app(config=None, modules=None): - if not modules: - modules = MODULES - app = Flask(APP_NAME) - app.config.from_pyfile(config) - configure_extensions(app) - configure_i18n(app) - configure_identity(app) - configure_blueprints(app, modules) - configure_logging(app) - configure_template_filters(app) - return app - - -def configure_extensions(app): - db.app = app - db.init_app(app) - mail.init_app(app) - cache.init_app(app) - celery.init_app(app) - rd.init_app(app) - - -def configure_i18n(app): - babel = Babel(app) - - @babel.localeselector - def get_locale(): - accept_languages = app.config.get('ACCEPT_LANGUAGES', ['en', 'zh']) - return request.accept_languages.best_match(accept_languages) - - -def configure_modules(app, modules): - for module, url_prefix in modules: - app.register_module(module, url_prefix=url_prefix) - - -def configure_blueprints(app, modules): - for module, url_prefix in modules: - app.register_blueprint(module, url_prefix=url_prefix) - - -def configure_identity(app): - principal = Principal(app) - @identity_loaded.connect_via(app) - def on_identity_loaded(sender, identity): - g.user = User.query.from_identity(identity) - - -def configure_logging(app): - hostname = os.uname()[1] - mail_handler = SMTPHandler( - app.config['MAIL_SERVER'], - app.config['DEFAULT_MAIL_SENDER'], - app.config['ADMINS'], - '[%s] CMDB API error' % hostname, - ( - app.config['MAIL_USERNAME'], - app.config['MAIL_PASSWORD'], - ) - ) - mail_formater = logging.Formatter( - "%(asctime)s %(levelname)s %(pathname)s %(lineno)d\n%(message)s") - mail_handler.setFormatter(mail_formater) - mail_handler.setLevel(logging.ERROR) - if not app.debug: - app.logger.addHandler(mail_handler) - formatter = logging.Formatter( - "%(asctime)s %(levelname)s %(pathname)s %(lineno)d - %(message)s") - log_file = app.config['LOG_PATH'] - file_handler = TimedRotatingFileHandler( - log_file, when='d', interval=1, backupCount=7) - file_handler.setLevel(getattr(logging, app.config['LOG_LEVEL'])) - file_handler.setFormatter(formatter) - app.logger.addHandler(file_handler) - app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL'])) - - -def configure_template_filters(app): - for name in dir(filters): - if callable(getattr(filters, name)): - app.add_template_filter(getattr(filters, name)) diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/api/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/api/app.py b/api/app.py new file mode 100644 index 0000000..a63f3bf --- /dev/null +++ b/api/app.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +"""The app module, containing the app factory function.""" +import os +import sys +import logging +from logging.handlers import RotatingFileHandler +from inspect import getmembers + +from flask import Flask +from flask import make_response, jsonify +from flask.blueprints import Blueprint +from flask.cli import click + +import api.views +from api.models.account import User +from api.flask_cas import CAS +from api.extensions import ( + bcrypt, + cors, + cache, + db, + login_manager, + migrate, + celery, + rd, +) + +HERE = os.path.abspath(os.path.dirname(__file__)) +PROJECT_ROOT = os.path.join(HERE, os.pardir) +API_PACKAGE = "api" + + +@login_manager.user_loader +def load_user(user_id): + """Load user by ID.""" + return User.get_by_id(int(user_id)) + + +class ReverseProxy(object): + """Wrap the application in this middleware and configure the + front-end server to add these headers, to let you quietly bind + this to a URL other than / and to an HTTP scheme that is + different than what is used locally. + + In nginx: + location /myprefix { + proxy_pass http://192.168.0.1:5001; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /myprefix; + } + + :param app: the WSGI application + """ + + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + script_name = environ.get('HTTP_X_SCRIPT_NAME', '') + if script_name: + environ['SCRIPT_NAME'] = script_name + path_info = environ['PATH_INFO'] + if path_info.startswith(script_name): + environ['PATH_INFO'] = path_info[len(script_name):] + + scheme = environ.get('HTTP_X_SCHEME', '') + if scheme: + environ['wsgi.url_scheme'] = scheme + return self.app(environ, start_response) + + +def create_app(config_object="{0}.settings".format(API_PACKAGE)): + """Create application factory, as explained here: http://flask.pocoo.org/docs/patterns/appfactories/. + + :param config_object: The configuration object to use. + """ + app = Flask(__name__.split(".")[0]) + app.config.from_object(config_object) + register_extensions(app) + register_blueprints(app) + register_error_handlers(app) + register_shell_context(app) + register_commands(app) + configure_logger(app) + CAS(app) + app.wsgi_app = ReverseProxy(app.wsgi_app) + return app + + +def register_extensions(app): + """Register Flask extensions.""" + bcrypt.init_app(app) + cache.init_app(app) + db.init_app(app) + cors.init_app(app) + login_manager.init_app(app) + migrate.init_app(app, db) + rd.init_app(app) + celery.conf.update(app.config) + + +def register_blueprints(app): + for item in getmembers(api.views): + if item[0].startswith("blueprint") and isinstance(item[1], Blueprint): + app.register_blueprint(item[1]) + + +def register_error_handlers(app): + """Register error handlers.""" + + def render_error(error): + """Render error template.""" + import traceback + app.logger.error(traceback.format_exc()) + error_code = getattr(error, "code", 500) + return make_response(jsonify(message=str(error)), error_code) + + for errcode in app.config.get("ERROR_CODES") or [400, 401, 403, 404, 405, 500, 502]: + app.errorhandler(errcode)(render_error) + app.handle_exception = render_error + + +def register_shell_context(app): + """Register shell context objects.""" + + def shell_context(): + """Shell context objects.""" + return {"db": db, "User": User} + + app.shell_context_processor(shell_context) + + +def register_commands(app): + """Register Click commands.""" + for root, _, files in os.walk(os.path.join(HERE, "commands")): + for filename in files: + if not filename.startswith("_") and filename.endswith("py"): + module_path = os.path.join(API_PACKAGE, root[root.index("commands"):]) + if module_path not in sys.path: + sys.path.insert(1, module_path) + command = __import__(os.path.splitext(filename)[0]) + func_list = [o[0] for o in getmembers(command) if isinstance(o[1], click.core.Command)] + for func_name in func_list: + app.cli.add_command(getattr(command, func_name)) + + +def configure_logger(app): + """Configure loggers.""" + handler = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter( + "%(asctime)s %(levelname)s %(pathname)s %(lineno)d - %(message)s") + + if app.debug: + handler.setFormatter(formatter) + app.logger.addHandler(handler) + + log_file = app.config['LOG_PATH'] + file_handler = RotatingFileHandler(log_file, + maxBytes=2 ** 30, + backupCount=7) + file_handler.setLevel(getattr(logging, app.config['LOG_LEVEL'])) + file_handler.setFormatter(formatter) + app.logger.addHandler(file_handler) + app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL'])) diff --git a/lib/__init__.py b/api/commands/__init__.py similarity index 63% rename from lib/__init__.py rename to api/commands/__init__.py index ef612ed..3b26f17 100644 --- a/lib/__init__.py +++ b/api/commands/__init__.py @@ -1,4 +1 @@ # -*- coding:utf-8 -*- - - -__all__ = [] \ No newline at end of file diff --git a/api/commands/common.py b/api/commands/common.py new file mode 100644 index 0000000..1abe98b --- /dev/null +++ b/api/commands/common.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +"""Click commands.""" +import os +from glob import glob +from subprocess import call + +import click +from flask import current_app +from flask.cli import with_appcontext +from werkzeug.exceptions import MethodNotAllowed, NotFound + +from api.extensions import db + +HERE = os.path.abspath(os.path.dirname(__file__)) +PROJECT_ROOT = os.path.join(HERE, os.pardir, os.pardir) +TEST_PATH = os.path.join(PROJECT_ROOT, "tests") + + +@click.command() +def test(): + """Run the tests.""" + import pytest + + rv = pytest.main([TEST_PATH, "--verbose"]) + exit(rv) + + +@click.command() +@click.option( + "-f", + "--fix-imports", + default=True, + is_flag=True, + help="Fix imports using isort, before linting", +) +@click.option( + "-c", + "--check", + default=False, + is_flag=True, + help="Don't make any changes to files, just confirm they are formatted correctly", +) +def lint(fix_imports, check): + """Lint and check code style with black, flake8 and isort.""" + skip = ["node_modules", "requirements", "migrations"] + root_files = glob("*.py") + root_directories = [ + name for name in next(os.walk("."))[1] if not name.startswith(".") + ] + files_and_directories = [ + arg for arg in root_files + root_directories if arg not in skip + ] + + def execute_tool(description, *args): + """Execute a checking tool with its arguments.""" + command_line = list(args) + files_and_directories + click.echo("{}: {}".format(description, " ".join(command_line))) + rv = call(command_line) + if rv != 0: + exit(rv) + + isort_args = ["-rc"] + black_args = [] + if check: + isort_args.append("-c") + black_args.append("--check") + if fix_imports: + execute_tool("Fixing import order", "isort", *isort_args) + execute_tool("Formatting style", "black", *black_args) + execute_tool("Checking code style", "flake8") + + +@click.command() +def clean(): + """Remove *.pyc and *.pyo files recursively starting at current directory. + + Borrowed from Flask-Script, converted to use Click. + """ + for dirpath, dirnames, filenames in os.walk("."): + for filename in filenames: + if filename.endswith(".pyc") or filename.endswith(".pyo"): + full_pathname = os.path.join(dirpath, filename) + click.echo("Removing {}".format(full_pathname)) + os.remove(full_pathname) + + +@click.command() +@click.option("--url", default=None, help="Url to test (ex. /static/image.png)") +@click.option( + "--order", default="rule", help="Property on Rule to order by (default: rule)" +) +@with_appcontext +def urls(url, order): + """Display all of the url matching routes for the project. + + Borrowed from Flask-Script, converted to use Click. + """ + rows = [] + column_headers = ("Rule", "Endpoint", "Arguments") + + if url: + try: + rule, arguments = current_app.url_map.bind("localhost").match( + url, return_rule=True + ) + rows.append((rule.rule, rule.endpoint, arguments)) + column_length = 3 + except (NotFound, MethodNotAllowed) as e: + rows.append(("<{}>".format(e), None, None)) + column_length = 1 + else: + rules = sorted( + current_app.url_map.iter_rules(), key=lambda rule: getattr(rule, order) + ) + for rule in rules: + rows.append((rule.rule, rule.endpoint, None)) + column_length = 2 + + str_template = "" + table_width = 0 + + if column_length >= 1: + max_rule_length = max(len(r[0]) for r in rows) + max_rule_length = max_rule_length if max_rule_length > 4 else 4 + str_template += "{:" + str(max_rule_length) + "}" + table_width += max_rule_length + + if column_length >= 2: + max_endpoint_length = max(len(str(r[1])) for r in rows) + max_endpoint_length = max_endpoint_length if max_endpoint_length > 8 else 8 + str_template += " {:" + str(max_endpoint_length) + "}" + table_width += 2 + max_endpoint_length + + if column_length >= 3: + max_arguments_length = max(len(str(r[2])) for r in rows) + max_arguments_length = max_arguments_length if max_arguments_length > 9 else 9 + str_template += " {:" + str(max_arguments_length) + "}" + table_width += 2 + max_arguments_length + + click.echo(str_template.format(*column_headers[:column_length])) + click.echo("-" * table_width) + + for row in rows: + click.echo(str_template.format(*row[:column_length])) + + +@click.command() +@with_appcontext +def db_setup(): + """create tables + """ + db.create_all() diff --git a/api/extensions.py b/api/extensions.py new file mode 100644 index 0000000..02706db --- /dev/null +++ b/api/extensions.py @@ -0,0 +1,21 @@ +# -*- coding:utf-8 -*- + + +from flask_bcrypt import Bcrypt +from flask_caching import Cache +from flask_login import LoginManager +from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy +from flask_cors import CORS +from celery import Celery + +from api.lib.utils import RedisHandler + +bcrypt = Bcrypt() +login_manager = LoginManager() +db = SQLAlchemy() +migrate = Migrate() +cache = Cache() +celery = Celery() +cors = CORS(supports_credentials=True) +rd = RedisHandler(prefix="CMDB_CI") # TODO diff --git a/api/flask_cas/__init__.py b/api/flask_cas/__init__.py new file mode 100644 index 0000000..dc31cf5 --- /dev/null +++ b/api/flask_cas/__init__.py @@ -0,0 +1,78 @@ +# -*- coding:utf-8 -*- + +""" +flask_cas.__init__ +""" + +import flask +from flask import current_app + +# Find the stack on which we want to store the database connection. +# Starting with Flask 0.9, the _app_ctx_stack is the correct one, +# before that we need to use the _request_ctx_stack. +try: + from flask import _app_ctx_stack as stack +except ImportError: + from flask import _request_ctx_stack as stack + +from api.flask_cas import routing + + +class CAS(object): + """ + Required Configs: + + |Key | + |----------------| + |CAS_SERVER | + |CAS_AFTER_LOGIN | + + Optional Configs: + + |Key | Default | + |-------------------------|----------------| + |CAS_TOKEN_SESSION_KEY | _CAS_TOKEN | + |CAS_USERNAME_SESSION_KEY | CAS_USERNAME | + |CAS_LOGIN_ROUTE | '/cas' | + |CAS_LOGOUT_ROUTE | '/cas/logout' | + |CAS_VALIDATE_ROUTE | '/cas/validate'| + """ + + def __init__(self, app=None, url_prefix=None): + self._app = app + if app is not None: + self.init_app(app, url_prefix) + + def init_app(self, app, url_prefix=None): + # Configuration defaults + app.config.setdefault('CAS_TOKEN_SESSION_KEY', '_CAS_TOKEN') + app.config.setdefault('CAS_USERNAME_SESSION_KEY', 'CAS_USERNAME') + app.config.setdefault('CAS_LOGIN_ROUTE', '/login') + app.config.setdefault('CAS_LOGOUT_ROUTE', '/logout') + app.config.setdefault('CAS_VALIDATE_ROUTE', '/serviceValidate') + # Register Blueprint + app.register_blueprint(routing.blueprint, url_prefix=url_prefix) + + # Use the newstyle teardown_appcontext if it's available, + # otherwise fall back to the request context + if hasattr(app, 'teardown_appcontext'): + app.teardown_appcontext(self.teardown) + else: + app.teardown_request(self.teardown) + + def teardown(self, exception): + ctx = stack.top + + @property + def app(self): + return self._app or current_app + + @property + def username(self): + return flask.session.get( + self.app.config['CAS_USERNAME_SESSION_KEY'], None) + + @property + def token(self): + return flask.session.get( + self.app.config['CAS_TOKEN_SESSION_KEY'], None) \ No newline at end of file diff --git a/api/flask_cas/cas_urls.py b/api/flask_cas/cas_urls.py new file mode 100644 index 0000000..34e15d3 --- /dev/null +++ b/api/flask_cas/cas_urls.py @@ -0,0 +1,122 @@ +# -*- coding:utf-8 -*- + +""" +flask_cas.cas_urls + +Functions for creating urls to access CAS. +""" +from six.moves.urllib.parse import quote +from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urljoin + + +def create_url(base, path=None, *query): + """ Create a url. + + Creates a url by combining base, path, and the query's list of + key/value pairs. Escaping is handled automatically. Any + key/value pair with a value that is None is ignored. + + Keyword arguments: + base -- The left most part of the url (ex. http://localhost:5000). + path -- The path after the base (ex. /foo/bar). + query -- A list of key value pairs (ex. [('key', 'value')]). + + Example usage: + >>> create_url( + ... 'http://localhost:5000', + ... 'foo/bar', + ... ('key1', 'value'), + ... ('key2', None), # Will not include None + ... ('url', 'http://example.com'), + ... ) + 'http://localhost:5000/foo/bar?key1=value&url=http%3A%2F%2Fexample.com' + """ + url = base + # Add the path to the url if it's not None. + if path is not None: + url = urljoin(url, quote(path)) + # Remove key/value pairs with None values. + query = filter(lambda pair: pair[1] is not None, query) + # Add the query string to the url + url = urljoin(url, '?{0}'.format(urlencode(list(query)))) + return url + + +def create_cas_login_url(cas_url, cas_route, service, + renew=None, gateway=None): + """ Create a CAS login URL . + + Keyword arguments: + cas_url -- The url to the CAS (ex. http://sso.pdx.edu) + cas_route -- The route where the CAS lives on server (ex. /cas) + service -- (ex. http://localhost:5000/login) + renew -- "true" or "false" + gateway -- "true" or "false" + + Example usage: + >>> create_cas_login_url( + ... 'http://sso.pdx.edu', + ... '/cas', + ... 'http://localhost:5000', + ... ) + 'http://sso.pdx.edu/cas?service=http%3A%2F%2Flocalhost%3A5000' + """ + return create_url( + cas_url, + cas_route, + ('service', service), + ('renew', renew), + ('gateway', gateway), + ) + + +def create_cas_logout_url(cas_url, cas_route, url=None): + """ Create a CAS logout URL. + + Keyword arguments: + cas_url -- The url to the CAS (ex. http://sso.pdx.edu) + cas_route -- The route where the CAS lives on server (ex. /cas/logout) + url -- (ex. http://localhost:5000/login) + + Example usage: + >>> create_cas_logout_url( + ... 'http://sso.pdx.edu', + ... '/cas/logout', + ... 'http://localhost:5000', + ... ) + 'http://sso.pdx.edu/cas/logout?url=http%3A%2F%2Flocalhost%3A5000' + """ + return create_url( + cas_url, + cas_route, + ('service', url), + ) + + +def create_cas_validate_url(cas_url, cas_route, service, ticket, + renew=None): + """ Create a CAS validate URL. + + Keyword arguments: + cas_url -- The url to the CAS (ex. http://sso.pdx.edu) + cas_route -- The route where the CAS lives on server (ex. /cas/validate) + service -- (ex. http://localhost:5000/login) + ticket -- (ex. 'ST-58274-x839euFek492ou832Eena7ee-cas') + renew -- "true" or "false" + + Example usage: + >>> create_cas_validate_url( + ... 'http://sso.pdx.edu', + ... '/cas/validate', + ... 'http://localhost:5000/login', + ... 'ST-58274-x839euFek492ou832Eena7ee-cas' + ... ) + """ + return create_url( + cas_url, + cas_route, + ('service', service), + ('ticket', ticket), + ('renew', renew), + ) \ No newline at end of file diff --git a/api/flask_cas/routing.py b/api/flask_cas/routing.py new file mode 100644 index 0000000..e726f32 --- /dev/null +++ b/api/flask_cas/routing.py @@ -0,0 +1,164 @@ +# -*- coding:utf-8 -*- + +import json + +import bs4 +from six.moves.urllib_request import urlopen + +from flask import Blueprint +from flask_login import login_user, logout_user +from flask import current_app, session, request, url_for, redirect + +from .cas_urls import create_cas_login_url +from .cas_urls import create_cas_logout_url +from .cas_urls import create_cas_validate_url + +from api.models.account import UserCache + +blueprint = Blueprint('cas', __name__) + + +@blueprint.route('/api/sso/login') +def login(): + """ + This route has two purposes. First, it is used by the user + to login. Second, it is used by the CAS to respond with the + `ticket` after the user logs in successfully. + + When the user accesses this url, they are redirected to the CAS + to login. If the login was successful, the CAS will respond to this + route with the ticket in the url. The ticket is then validated. + If validation was successful the logged in username is saved in + the user's session under the key `CAS_USERNAME_SESSION_KEY`. + """ + + cas_token_session_key = current_app.config['CAS_TOKEN_SESSION_KEY'] + if request.values.get("next"): + session["next"] = request.values.get("next") + + _service = url_for('cas.login', _external=True, next=session["next"]) \ + if session.get("next") else url_for('cas.login', _external=True) + redirect_url = create_cas_login_url( + current_app.config['CAS_SERVER'], + current_app.config['CAS_LOGIN_ROUTE'], + _service) + + if 'ticket' in request.args: + session[cas_token_session_key] = request.args.get('ticket') + + if request.args.get('ticket'): + + if validate(request.args['ticket']): + redirect_url = session.get("next") or \ + current_app.config.get("CAS_AFTER_LOGIN") + username = session.get("CAS_USERNAME") + user = UserCache.get(username) + login_user(user) + + session.permanent = True + + else: + del session[cas_token_session_key] + redirect_url = create_cas_login_url( + current_app.config['CAS_SERVER'], + current_app.config['CAS_LOGIN_ROUTE'], + url_for('cas.login', _external=True), + renew=True) + current_app.logger.info("redirect to: {0}".format(redirect_url)) + return redirect(redirect_url) + + +@blueprint.route('/api/sso/logout') +def logout(): + """ + When the user accesses this route they are logged out. + """ + + cas_username_session_key = current_app.config['CAS_USERNAME_SESSION_KEY'] + cas_token_session_key = current_app.config['CAS_TOKEN_SESSION_KEY'] + + cas_username_session_key in session and session.pop(cas_username_session_key) + "acl" in session and session.pop("acl") + "uid" in session and session.pop("uid") + cas_token_session_key in session and session.pop(cas_token_session_key) + "next" in session and session.pop("next") + + redirect_url = create_cas_logout_url( + current_app.config['CAS_SERVER'], + current_app.config['CAS_LOGOUT_ROUTE'], + url_for('cas.login', _external=True, next=request.referrer)) + + logout_user() + + current_app.logger.debug('Redirecting to: {0}'.format(redirect_url)) + + return redirect(redirect_url) + + +def validate(ticket): + """ + Will attempt to validate the ticket. If validation fails, then False + is returned. If validation is successful, then True is returned + and the validated username is saved in the session under the + key `CAS_USERNAME_SESSION_KEY`. + """ + + cas_username_session_key = current_app.config['CAS_USERNAME_SESSION_KEY'] + + current_app.logger.debug("validating token {0}".format(ticket)) + + cas_validate_url = create_cas_validate_url( + current_app.config['CAS_VALIDATE_SERVER'], + current_app.config['CAS_VALIDATE_ROUTE'], + url_for('cas.login', _external=True), + ticket) + + current_app.logger.debug("Making GET request to {0}".format(cas_validate_url)) + + try: + response = urlopen(cas_validate_url).read() + ticketid = _parse_tag(response, "cas:user") + strs = [s.strip() for s in ticketid.split('|') if s.strip()] + username, is_valid = None, False + if len(strs) == 1: + username = strs[0] + is_valid = True + user_info = json.loads(_parse_tag(response, "cas:other")) + current_app.logger.info(user_info) + except ValueError: + current_app.logger.error("CAS returned unexpected result") + is_valid = False + return is_valid + + if is_valid: + current_app.logger.debug("valid") + session[cas_username_session_key] = username + user = UserCache.get(username) + session["acl"] = dict(uid=user_info.get("uuid"), + avatar=user.avatar if user else user_info.get("avatar"), + userId=user_info.get("id"), + userName=user_info.get("name"), + nickName=user_info.get("nickname"), + parentRoles=user_info.get("parents"), + childRoles=user_info.get("children"), + roleName=user_info.get("role")) + session["uid"] = user_info.get("uuid") + current_app.logger.debug(session) + current_app.logger.debug(request.url) + else: + current_app.logger.debug("invalid") + + return is_valid + + +def _parse_tag(string, tag): + """ + Used for parsing xml. Search string for the first occurence of + ..... and return text (stripped of leading and tailing + whitespace) between tags. Return "" if tag not found. + """ + soup = bs4.BeautifulSoup(string) + + if soup.find(tag) is None: + return '' + return soup.find(tag).string.strip() diff --git a/api/lib/__init__.py b/api/lib/__init__.py new file mode 100644 index 0000000..3b26f17 --- /dev/null +++ b/api/lib/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/api/lib/cmdb/__init__.py b/api/lib/cmdb/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/api/lib/cmdb/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/api/lib/cmdb/attribute.py b/api/lib/cmdb/attribute.py new file mode 100644 index 0000000..f36de13 --- /dev/null +++ b/api/lib/cmdb/attribute.py @@ -0,0 +1,150 @@ +# -*- coding:utf-8 -*- + +from flask import current_app +from flask import abort + +from api.extensions import db +from api.models.cmdb import Attribute +from api.models.cmdb import CITypeAttribute +from api.models.cmdb import PreferenceShowAttributes +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.const import type_map +from api.lib.decorator import kwargs_required + + +class AttributeManager(object): + """ + CI attributes manager + """ + + def __init__(self): + pass + + @staticmethod + def get_choice_values(attr_id, value_type): + choice_table = type_map.get("choice").get(value_type) + choice_values = choice_table.get_by(fl=["value"], attr_id=attr_id) + return [choice_value["value"] for choice_value in choice_values] + + @staticmethod + def _add_choice_values(_id, value_type, choice_values): + choice_table = type_map.get("choice").get(value_type) + + db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() + db.session.flush() + choice_values = choice_values + for v in choice_values: + table = choice_table(attr_id=_id, value=v) + db.session.add(table) + db.session.flush() + + def get_attributes(self, name=None): + """ + :param name: + :return: attribute, if name is None, then return all attributes + """ + attrs = Attribute.get_by_like(name=name) if name is not None else Attribute.get_by() + res = list() + for attr in attrs: + attr["is_choice"] and attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"]))) + res.append(attr) + return res + + def get_attribute_by_name(self, name): + attr = Attribute.get_by(name=name, first=True) + if attr and attr["is_choice"]: + attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"]))) + return attr + + def get_attribute_by_alias(self, alias): + attr = Attribute.get_by(alias=alias, first=True) + if attr and attr["is_choice"]: + attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"]))) + return attr + + def get_attribute_by_id(self, _id): + attr = Attribute.get_by_id(_id).to_dict() + if attr and attr["is_choice"]: + attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"]))) + return attr + + def get_attribute(self, key): + attr = AttributeCache.get(key).to_dict() + if attr and attr["is_choice"]: + attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"]))) + return attr + + @classmethod + @kwargs_required("name") + def add(cls, **kwargs): + choice_value = kwargs.pop("choice_value", []) + kwargs.pop("is_choice", None) + is_choice = True if choice_value else False + name = kwargs.pop("name") + alias = kwargs.pop("alias", "") + alias = name if not alias else alias + Attribute.get_by(name=name, first=True) and abort(400, "attribute {0} is already existed".format(name)) + + attr = Attribute.create(flush=True, + name=name, + alias=alias, + is_choice=is_choice, + **kwargs) + + if choice_value: + cls._add_choice_values(attr.id, attr.value_type, choice_value) + + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("add attribute error, {0}".format(str(e))) + return abort(400, "add attribute <{0}> failed".format(name)) + + AttributeCache.clean(attr) + + return attr.id + + def update(self, _id, **kwargs): + attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id)) + + choice_value = kwargs.pop("choice_value", False) + is_choice = True if choice_value else False + + attr.update(flush=True, **kwargs) + + if is_choice: + self._add_choice_values(attr.id, attr.value_type, choice_value) + + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("update attribute error, {0}".format(str(e))) + return abort(400, "update attribute <{0}> failed".format(_id)) + + AttributeCache.clean(attr) + + return attr.id + + @staticmethod + def delete(_id): + attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id)) + name = attr.name + + if attr.is_choice: + choice_table = type_map["choice"].get(attr.value_type) + db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() # FIXME: session conflict + db.session.flush() + + AttributeCache.clean(attr) + + attr.soft_delete() + + for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False): + i.soft_delete() + + for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False): + i.soft_delete() + + return name diff --git a/api/lib/cmdb/cache.py b/api/lib/cmdb/cache.py new file mode 100644 index 0000000..08f25b9 --- /dev/null +++ b/api/lib/cmdb/cache.py @@ -0,0 +1,138 @@ +# -*- coding:utf-8 -*- + +import six +if six.PY2: + import sys + reload(sys) + sys.setdefaultencoding("utf-8") + +from api.extensions import cache +from api.models.cmdb import Attribute +from api.models.cmdb import CIType +from api.models.cmdb import CITypeAttribute +from api.models.cmdb import RelationType + + +class AttributeCache(object): + @classmethod + def get(cls, key): + if key is None: + return + attr = cache.get('Field::Name::{0}'.format(key)) \ + or cache.get('Field::ID::{0}'.format(key)) \ + or cache.get('Field::Alias::{0}'.format(key)) + + if attr is None: + attr = Attribute.get_by(name=key, first=True, to_dict=False) \ + or Attribute.get_by_id(key) \ + or Attribute.get_by(alias=key, first=True, to_dict=False) + if attr is not None: + cls.set(attr) + return attr + + @classmethod + def set(cls, attr): + cache.set('Field::ID::{0}'.format(attr.id), attr) + cache.set('Field::Name::{0}'.format(attr.name), attr) + cache.set('Field::Alias::{0}'.format(attr.alias), attr) + + @classmethod + def clean(cls, attr): + cache.delete('Field::ID::{0}'.format(attr.id)) + cache.delete('Field::Name::{0}'.format(attr.name)) + cache.delete('Field::Alias::{0}'.format(attr.alias)) + + +class CITypeCache(object): + @classmethod + def get(cls, key): + if key is None: + return + ct = cache.get("CIType::ID::{0}".format(key)) or \ + cache.get("CIType::Name::{0}".format(key)) or \ + cache.get("CIType::Alias::{0}".format(key)) + if ct is None: + ct = CIType.get_by(name=key, first=True, to_dict=False) or \ + CIType.get_by_id(key) or \ + CIType.get_by(alias=key, first=True, to_dict=False) + if ct is not None: + cls.set(ct) + return ct + + @classmethod + def set(cls, ct): + cache.set("CIType::Name::{0}".format(ct.name), ct) + cache.set("CIType::ID::{0}".format(ct.id), ct) + cache.set("CIType::Alias::{0}".format(ct.alias), ct) + + @classmethod + def clean(cls, key): + ct = cls.get(key) + if ct is not None: + cache.delete("CIType::Name::{0}".format(ct.name)) + cache.delete("CIType::ID::{0}".format(ct.id)) + cache.delete("CIType::Alias::{0}".format(ct.alias)) + + +class RelationTypeCache(object): + @classmethod + def get(cls, key): + if key is None: + return + ct = cache.get("RelationType::ID::{0}".format(key)) or \ + cache.get("RelationType::Name::{0}".format(key)) + if ct is None: + ct = RelationType.get_by(name=key, first=True, to_dict=False) or RelationType.get_by_id(key) + if ct is not None: + cls.set(ct) + return ct + + @classmethod + def set(cls, ct): + cache.set("RelationType::Name::{0}".format(ct.name), ct) + cache.set("RelationType::ID::{0}".format(ct.id), ct) + + @classmethod + def clean(cls, key): + ct = cls.get(key) + if ct is not None: + cache.delete("RelationType::Name::{0}".format(ct.name)) + cache.delete("RelationType::ID::{0}".format(ct.id)) + + +class CITypeAttributeCache(object): + """ + key is type_id or type_name + """ + + @classmethod + def get(cls, key): + if key is None: + return + + attrs = cache.get("CITypeAttribute::Name::{0}".format(key)) \ + or cache.get("CITypeAttribute::ID::{0}".format(key)) + if not attrs: + attrs = CITypeAttribute.get_by(type_id=key, to_dict=False) + if not attrs: + ci_type = CIType.get_by(name=key, first=True, to_dict=False) + if ci_type is not None: + attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False) + if attrs is not None: + cls.set(key, attrs) + return attrs + + @classmethod + def set(cls, key, values): + ci_type = CITypeCache.get(key) + if ci_type is not None: + cache.set("CITypeAttribute::ID::{0}".format(ci_type.id), values) + cache.set("CITypeAttribute::Name::{0}".format(ci_type.name), values) + + @classmethod + def clean(cls, key): + ci_type = CITypeCache.get(key) + attrs = cls.get(key) + if attrs is not None and ci_type: + cache.delete("CITypeAttribute::ID::{0}".format(ci_type.id)) + cache.delete("CITypeAttribute::Name::{0}".format(ci_type.name)) diff --git a/api/lib/cmdb/ci.py b/api/lib/cmdb/ci.py new file mode 100644 index 0000000..e7f90c5 --- /dev/null +++ b/api/lib/cmdb/ci.py @@ -0,0 +1,552 @@ +# -*- coding:utf-8 -*- + + +import datetime +import json + +from flask import abort +from flask import current_app +from werkzeug.exceptions import BadRequest + +from api.extensions import db +from api.extensions import rd +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.cache import RelationTypeCache +from api.lib.cmdb.ci_type import CITypeAttributeManager +from api.lib.cmdb.ci_type import CITypeManager +from api.lib.cmdb.const import CMDB_QUEUE +from api.lib.cmdb.const import ExistPolicy +from api.lib.cmdb.const import OperateType +from api.lib.cmdb.const import RetKey +from api.lib.cmdb.const import TableMap +from api.lib.cmdb.const import type_map +from api.lib.cmdb.history import AttributeHistoryManger +from api.lib.cmdb.history import CIRelationHistoryManager +from api.lib.cmdb.query_sql import QUERY_CIS_BY_IDS +from api.lib.cmdb.query_sql import QUERY_CIS_BY_VALUE_TABLE +from api.lib.cmdb.value import AttributeValueManager +from api.lib.decorator import kwargs_required +from api.lib.utils import handle_arg_list +from api.models.cmdb import CI +from api.models.cmdb import CIRelation +from api.models.cmdb import CITypeAttribute +from api.tasks.cmdb import ci_cache +from api.tasks.cmdb import ci_delete + + +class CIManager(object): + """ manage CI interface + """ + + def __init__(self): + pass + + @staticmethod + def get_type_name(ci_id): + ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id)) + return CITypeCache.get(ci.type_id).name + + @staticmethod + def confirm_ci_existed(ci_id): + CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id)) + + @classmethod + def get_ci_by_id(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_children=True): + """ + + :param ci_id: + :param ret_key: name, id, or alias + :param fields: attribute list + :param need_children: + :return: + """ + + ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id)) + + res = dict() + + if need_children: + children = CIRelationManager.get_children(ci_id, ret_key=ret_key) # one floor + res.update(children) + + ci_type = CITypeCache.get(ci.type_id) + res["ci_type"] = ci_type.name + + res.update(cls.get_cis_by_ids([str(ci_id)], fields=fields, ret_key=ret_key)) + + res['_type'] = ci_type.id + res['_id'] = ci_id + + return res + + @staticmethod + def get_ci_by_id_from_db(ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False): + """ + + :param ci_id: + :param ret_key: name, id or alias + :param fields: list + :param need_children: + :param use_master: whether to use master db + :return: + """ + + ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id)) + + res = dict() + + if need_children: + children = CIRelationManager.get_children(ci_id, ret_key=ret_key) # one floor + res.update(children) + + ci_type = CITypeCache.get(ci.type_id) + res["ci_type"] = ci_type.name + + fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields + + _res = AttributeValueManager().get_attr_values(fields, ci_id, ret_key=ret_key, use_master=use_master) + res.update(_res) + + res['_type'] = ci_type.id + res['_id'] = ci_id + + return res + + def get_ci_by_ids(self, ci_id_list, ret_key=RetKey.NAME, fields=None): + return [self.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) for ci_id in ci_id_list] + + @classmethod + def get_cis_by_type(cls, type_id, ret_key=RetKey.NAME, fields="", page=1, per_page=None): + cis = db.session.query(CI.id).filter(CI.type_id == type_id).filter(CI.deleted.is_(False)) + numfound = cis.count() + + cis = cis.offset((page - 1) * per_page).limit(per_page) + ci_ids = [str(ci.id) for ci in cis] + res = cls.get_cis_by_ids(ci_ids, ret_key, fields) + + return numfound, page, res + + @staticmethod + def ci_is_exist(unique_key, unique_value): + """ + + :param unique_key: is a attribute + :param unique_value: + :return: + """ + value_table = TableMap(attr_name=unique_key.name).table + unique = value_table.get_by(attr_id=unique_key.id, + value=unique_value, + to_dict=False, + first=True) + if unique: + return CI.get_by_id(unique.ci_id) + + @staticmethod + def _delete_ci_by_id(ci_id): + ci = CI.get_by_id(ci_id) + ci.delete() # TODO: soft delete + + @classmethod + def add(cls, ci_type_name, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=ExistPolicy.IGNORE, **ci_dict): + """ + + :param ci_type_name: + :param exist_policy: replace or reject or need + :param _no_attribute_policy: ignore or reject + :param ci_dict: + :return: + """ + + ci_type = CITypeManager.check_is_existed(ci_type_name) + + unique_key = AttributeCache.get(ci_type.unique_id) or abort(400, 'illegality unique attribute') + + unique_value = ci_dict.get(unique_key.name) or \ + ci_dict.get(unique_key.alias) or \ + ci_dict.get(unique_key.id) or \ + abort(400, '{0} missing'.format(unique_key.name)) + + existed = cls.ci_is_exist(unique_key, unique_value) + if existed is not None: + if exist_policy == ExistPolicy.REJECT: + return abort(400, 'CI is already existed') + if existed.type_id != ci_type.id: + existed.update(type_id=ci_type.id) + ci = existed + else: + if exist_policy == ExistPolicy.NEED: + return abort(404, 'CI <{0}> does not exist'.format(unique_value)) + ci = CI.create(type_id=ci_type.id) + + value_manager = AttributeValueManager() + for p, v in ci_dict.items(): + try: + value_manager.create_or_update_attr_value(p, v, ci.id, _no_attribute_policy) + except BadRequest as e: + if existed is None: + cls.delete(ci.id) + raise e + + ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) + + return ci.id + + def update(self, ci_id, **ci_dict): + self.confirm_ci_existed(ci_id) + value_manager = AttributeValueManager() + for p, v in ci_dict.items(): + try: + value_manager.create_or_update_attr_value(p, v, ci_id) + except BadRequest as e: + raise e + + ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + + @staticmethod + def update_unique_value(ci_id, unique_name, unique_value): + CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not found".format(ci_id)) + + AttributeValueManager().create_or_update_attr_value(unique_name, unique_value, ci_id) + + ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + + @staticmethod + def delete(ci_id): + ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not found".format(ci_id)) + + attrs = CITypeAttribute.get_by(type_id=ci.type_id, to_dict=False) + attr_names = set([AttributeCache.get(attr.attr_id).name for attr in attrs]) + for attr_name in attr_names: + value_table = TableMap(attr_name=attr_name).table + for item in value_table.get_by(ci_id=ci_id, to_dict=False): + item.delete() + + for item in CIRelation.get_by(first_ci_id=ci_id, to_dict=False): + item.delete() + + for item in CIRelation.get_by(second_ci_id=ci_id, to_dict=False): + item.delete() + + ci.delete() # TODO: soft delete + + AttributeHistoryManger.add(ci_id, [(None, OperateType.DELETE, None, None)]) + + ci_delete.apply_async([ci.id], queue=CMDB_QUEUE) + + return ci_id + + @staticmethod + def add_heartbeat(ci_type, unique_value): + ci_type = CITypeManager().check_is_existed(ci_type) + + unique_key = AttributeCache.get(ci_type.unique_id) + value_table = TableMap(attr_name=unique_key.name).table + + v = value_table.get_by(attr_id=unique_key.id, + value=unique_value, + to_dict=False, + first=True) \ + or abort(404, "not found") + + ci = CI.get_by_id(v.ci_id) or abort(404, "CI <{0}> is not found".format(v.ci_id)) + + ci.update(heartbeat=datetime.datetime.now()) + + @classmethod + @kwargs_required("type_id", "page") + def get_heartbeat(cls, **kwargs): + query = db.session.query(CI.id, CI.heartbeat).filter(CI.deleted.is_(False)) + + expire = datetime.datetime.now() - datetime.timedelta(minutes=72) + type_ids = handle_arg_list(kwargs["type_id"]) + + query = query.filter(CI.type_id.in_(type_ids)) + + page = kwargs.get("page") + agent_status = kwargs.get("agent_status") + if agent_status == -1: + query = query.filter(CI.heartbeat.is_(None)) + elif agent_status == 0: + query = query.filter(CI.heartbeat <= expire) + elif agent_status == 1: + query = query.filter(CI.heartbeat > expire) + + numfound = query.count() + per_page_count = current_app.config.get("DEFAULT_PAGE_COUNT") + cis = query.offset((page - 1) * per_page_count).limit(per_page_count).all() + ci_ids = [ci.id for ci in cis] + heartbeat_dict = {} + for ci in cis: + if agent_status is not None: + heartbeat_dict[ci.id] = agent_status + else: + if ci.heartbeat is None: + heartbeat_dict[ci.id] = -1 + elif ci.heartbeat <= expire: + heartbeat_dict[ci.id] = 0 + else: + heartbeat_dict[ci.id] = 1 + current_app.logger.debug(heartbeat_dict) + ci_ids = list(map(str, ci_ids)) + res = cls.get_cis_by_ids(ci_ids, fields=["hostname", "private_ip"]) + result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"), + heartbeat_dict.get(i.get("_id"))) for i in res + if i.get("private_ip")] + return numfound, result + + @staticmethod + def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None): + res = rd.get(ci_ids) + if res is not None and None not in res and ret_key == RetKey.NAME: + res = list(map(json.loads, res)) + if not fields: + return res + else: + _res = [] + for d in res: + _d = dict() + _d["_id"], _d["_type"] = d.get("_id"), d.get("_type") + _d["ci_type"] = d.get("ci_type") + for field in fields: + _d[field] = d.get(field) + _res.append(_d) + return _res + + @staticmethod + def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None): + if not fields: + filter_fields_sql = "" + else: + _fields = list() + for field in fields: + attr = AttributeCache.get(field) + if attr is not None: + _fields.append(str(attr.id)) + filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields)) + + ci_ids = ",".join(ci_ids) + if value_tables is None: + value_tables = type_map["table_name"].values() + + value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids) + for value_table in value_tables]) + query_sql = QUERY_CIS_BY_IDS.format(filter_fields_sql, value_sql) + # current_app.logger.debug(query_sql) + cis = db.session.execute(query_sql).fetchall() + ci_set = set() + res = list() + ci_dict = dict() + for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis: + if ci_id not in ci_set: + ci_dict = dict() + ci_type = CITypeCache.get(type_id) + ci_dict["_id"] = ci_id + ci_dict["_type"] = type_id + ci_dict["ci_type"] = ci_type.name + ci_dict["ci_type_alias"] = ci_type.alias + ci_set.add(ci_id) + res.append(ci_dict) + + if ret_key == RetKey.NAME: + attr_key = attr_name + elif ret_key == RetKey.ALIAS: + attr_key = attr_alias + elif ret_key == RetKey.ID: + attr_key = attr_id + else: + return abort(400, "invalid ret key") + + value = type_map["serialize2"][value_type](value) + if is_list: + ci_dict.setdefault(attr_key, []).append(value) + else: + ci_dict[attr_key] = value + + return res + + @classmethod + def get_cis_by_ids(cls, ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None): + """ + + :param ci_ids: list of CI instance ID, eg. ['1', '2'] + :param ret_key: name, id or alias + :param fields: + :param value_tables: + :return: + """ + + if not ci_ids: + return [] + + fields = [] if fields is None or not isinstance(fields, list) else fields + + ci_id_tuple = tuple(map(int, ci_ids)) + res = cls._get_cis_from_cache(ci_id_tuple, ret_key, fields) + if res is not None: + return res + + current_app.logger.warning("cache not hit...............") + return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables) + + +class CIRelationManager(object): + """ + Manage relation between CIs + """ + + def __init__(self): + pass + + @staticmethod + def _get_default_relation_type(): + return RelationTypeCache.get("contain").id # FIXME + + @classmethod + def get_children(cls, ci_id, ret_key=RetKey.NAME): + second_cis = CIRelation.get_by(first_ci_id=ci_id, to_dict=False) + second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis) + ci_type2ci_ids = dict() + for ci_id in second_ci_ids: + type_id = CI.get_by_id(ci_id).type_id + ci_type2ci_ids.setdefault(type_id, []).append(ci_id) + + res = {} + for type_id in ci_type2ci_ids: + ci_type = CITypeCache.get(type_id) + children = CIManager.get_cis_by_ids(list(map(str, ci_type2ci_ids[type_id])), ret_key=ret_key) + res[ci_type.name] = children + return res + + def get_second_cis(self, first_ci_id, relation_type_id=None, page=1, per_page=None, **kwargs): + second_cis = db.session.query(CI.id).filter(CI.deleted.is_(False)).join( + CIRelation, CIRelation.second_ci_id == CI.id).filter( + CIRelation.first_ci_id == first_ci_id) + + if relation_type_id is not None: + second_cis = second_cis.filter(CIRelation.relation_type_id == relation_type_id) + + if kwargs: # TODO: special for devices + second_cis = self._query_wrap_for_device(second_cis, **kwargs) + + numfound = second_cis.count() + if per_page != "all": + second_cis = second_cis.offset((page - 1) * per_page).limit(per_page).all() + ci_ids = [str(son.id) for son in second_cis] + result = CIManager.get_cis_by_ids(ci_ids) + + return numfound, len(ci_ids), result + + @staticmethod + def _sort_handler(sort_by, query_sql): + + if sort_by.startswith("+"): + sort_type = "asc" + sort_by = sort_by[1:] + elif sort_by.startswith("-"): + sort_type = "desc" + sort_by = sort_by[1:] + else: + sort_type = "asc" + attr = AttributeCache.get(sort_by) + if attr is None: + return query_sql + + attr_id = attr.id + value_table = TableMap(attr_name=sort_by).table + + ci_table = query_sql.subquery() + query_sql = db.session.query(ci_table.c.id, value_table.value).join( + value_table, value_table.ci_id == ci_table.c.id).filter( + value_table.attr_id == attr_id).filter(ci_table.deleted.is_(False)).order_by( + getattr(value_table.value, sort_type)()) + + return query_sql + + def _query_wrap_for_device(self, query_sql, **kwargs): + _type = kwargs.pop("_type", False) or kwargs.pop("type", False) or kwargs.pop("ci_type", False) + if _type: + ci_type = CITypeCache.get(_type) + if ci_type is None: + return + query_sql = query_sql.filter(CI.type_id == ci_type.id) + + for k, v in kwargs.items(): + attr = AttributeCache.get(k) + if attr is None: + continue + + value_table = TableMap(attr_name=k).table + ci_table = query_sql.subquery() + query_sql = db.session.query(ci_table.c.id).join( + value_table, value_table.ci_id == ci_table.c.id).filter( + value_table.attr_id == attr.id).filter(ci_table.deleted.is_(False)).filter( + value_table.value.ilike(v.replace("*", "%"))) + + # current_app.logger.debug(query_sql) + sort_by = kwargs.pop("sort", "") + if sort_by: + query_sql = self._sort_handler(sort_by, query_sql) + + return query_sql + + @classmethod + def get_first_cis(cls, second_ci, relation_type_id=None, page=1, per_page=None): + first_cis = db.session.query(CIRelation.first_ci_id).filter( + CIRelation.second_ci_id == second_ci).filter(CIRelation.deleted.is_(False)) + if relation_type_id is not None: + first_cis = first_cis.filter(CIRelation.relation_type_id == relation_type_id) + + numfound = first_cis.count() + if per_page != "all": + first_cis = first_cis.offset((page - 1) * per_page).limit(per_page).all() + + first_ci_ids = [str(first_ci.first_ci_id) for first_ci in first_cis] + result = CIManager.get_cis_by_ids(first_ci_ids) + + return numfound, len(first_ci_ids), result + + @classmethod + def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None): + + relation_type_id = relation_type_id or cls._get_default_relation_type() + + CIManager.confirm_ci_existed(first_ci_id) + CIManager.confirm_ci_existed(second_ci_id) + + existed = CIRelation.get_by(first_ci_id=first_ci_id, + second_ci_id=second_ci_id, + to_dict=False, + first=True) + if existed is not None: + if existed.relation_type_id != relation_type_id: + existed.update(relation_type_id=relation_type_id) + CIRelationHistoryManager().add(existed, OperateType.UPDATE) + else: + existed = CIRelation.create(first_ci_id=first_ci_id, + second_ci_id=second_ci_id, + relation_type_id=relation_type_id) + CIRelationHistoryManager().add(existed, OperateType.ADD) + if more is not None: + existed.upadte(more=more) + + return existed.id + + @staticmethod + def delete(cr_id): + cr = CIRelation.get_by_id(cr_id) or abort(404, "CIRelation <{0}> is not existed".format(cr_id)) + cr.soft_delete() + + his_manager = CIRelationHistoryManager() + his_manager.add(cr, operate_type=OperateType.DELETE) + + return cr_id + + @classmethod + def delete_2(cls, first_ci_id, second_ci_id): + cr = CIRelation.get_by(first_ci_id=first_ci_id, + second_ci_id=second_ci_id, + to_dict=False, + first=True) + return cls.delete(cr.cr_id) diff --git a/api/lib/cmdb/ci_type.py b/api/lib/cmdb/ci_type.py new file mode 100644 index 0000000..c4b2b23 --- /dev/null +++ b/api/lib/cmdb/ci_type.py @@ -0,0 +1,392 @@ +# -*- coding:utf-8 -*- + + +from flask import current_app +from flask import abort + +from api.models.cmdb import CITypeAttribute +from api.models.cmdb import CIType +from api.models.cmdb import CITypeGroup +from api.models.cmdb import CITypeGroupItem +from api.models.cmdb import CITypeRelation +from api.models.cmdb import CITypeAttributeGroup +from api.models.cmdb import CITypeAttributeGroupItem +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import CITypeAttributeCache +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.attribute import AttributeManager +from api.lib.decorator import kwargs_required + + +class CITypeManager(object): + """ + manage CIType + """ + + def __init__(self): + pass + + @staticmethod + def get_name_by_id(type_id): + return CITypeCache.get(type_id).name + + @staticmethod + def check_is_existed(key): + return CITypeCache.get(key) or abort(404, "CIType <{0}> is not existed".format(key)) + + @staticmethod + def get_ci_types(type_name=None): + ci_types = CIType.get_by() if type_name is None else CIType.get_by_like(name=type_name) + res = list() + for type_dict in ci_types: + type_dict["unique_key"] = AttributeCache.get(type_dict["unique_id"]).name + res.append(type_dict) + return res + + @staticmethod + def query(_type): + ci_type = CITypeCache.get(_type) or abort(404, "CIType <{0}> is not found".format(_type)) + return ci_type.to_dict() + + @classmethod + @kwargs_required("name") + def add(cls, **kwargs): + unique_key = kwargs.pop("unique_key", None) + unique_key = AttributeCache.get(unique_key) or abort(404, "Unique key is not defined") + + CIType.get_by(name=kwargs['name'], first=True) and \ + abort(404, "CIType <{0}> is already existed".format(kwargs.get("name"))) + + kwargs["alias"] = kwargs["name"] if not kwargs.get("alias") else kwargs["alias"] + + kwargs["unique_id"] = unique_key.id + ci_type = CIType.create(**kwargs) + + CITypeAttributeManager.add(ci_type.id, [unique_key.id], is_required=True) + + CITypeCache.clean(ci_type.name) + + return ci_type.id + + @classmethod + def update(cls, type_id, **kwargs): + + ci_type = cls.check_is_existed(type_id) + + unique_key = kwargs.pop("unique_key", None) + unique_key = AttributeCache.get(unique_key) + if unique_key is not None: + kwargs["unique_id"] = unique_key.id + type_attr = CITypeAttribute.get_by(type_id=type_id, + attr_id=unique_key.id, + first=True, + to_dict=False) + if type_attr is None: + CITypeAttributeManager.add(type_id, [unique_key.id], is_required=True) + + ci_type.update(**kwargs) + + CITypeCache.clean(type_id) + + return type_id + + @classmethod + def set_enabled(cls, type_id, enabled=True): + ci_type = cls.check_is_existed(type_id) + ci_type.update(enabled=enabled) + return type_id + + @classmethod + def delete(cls, type_id): + ci_type = cls.check_is_existed(type_id) + ci_type.soft_delete() + + CITypeCache.clean(type_id) + + +class CITypeGroupManager(object): + @staticmethod + def get(need_other=None): + groups = CITypeGroup.get_by() + group_types = set() + for group in groups: + for t in sorted(CITypeGroupItem.get_by(group_id=group['id']), key=lambda x: x['order']): + group.setdefault("ci_types", []).append(CITypeCache.get(t['type_id']).to_dict()) + group_types.add(t["type_id"]) + + if need_other: + ci_types = CITypeManager.get_ci_types() + other_types = dict(ci_types=[ci_type for ci_type in ci_types if ci_type["id"] not in group_types]) + groups.append(other_types) + + return groups + + @staticmethod + def add(name): + CITypeGroup.get_by(name=name, first=True) and abort(400, "Group {0} does exist".format(name)) + return CITypeGroup.create(name=name) + + @staticmethod + def update(gid, name, type_ids): + """ + update all + :param gid: + :param name: + :param type_ids: + :return: + """ + existed = CITypeGroup.get_by_id(gid) or abort(404, "Group <{0}> does not exist".format(gid)) + if name is not None: + existed.update(name=name) + + for idx, type_id in enumerate(type_ids): + + item = CITypeGroupItem.get_by(group_id=gid, type_id=type_id, first=True, to_dict=False) + if item is not None: + item.update(order=idx) + else: + CITypeGroupItem.create(group_id=gid, type_id=type_id, order=idx) + + @staticmethod + def delete(gid): + existed = CITypeGroup.get_by_id(gid) or abort(404, "Group <{0}> does not exist".format(gid)) + + items = CITypeGroupItem.get_by(group_id=gid, to_dict=False) + for item in items: + item.soft_delete() + + existed.soft_delete() + + +class CITypeAttributeManager(object): + """ + manage CIType's attributes, include query, add, update, delete + """ + + def __init__(self): + pass + + @staticmethod + def get_attr_names_by_type_id(type_id): + return [AttributeCache.get(attr.attr_id).name for attr in CITypeAttributeCache.get(type_id)] + + @staticmethod + def get_attributes_by_type_id(type_id): + attrs = CITypeAttributeCache.get(type_id) + result = list() + for attr in sorted(attrs, key=lambda x: (x.order, x.id)): + attr_dict = AttributeManager().get_attribute(attr.attr_id) + attr_dict["is_required"] = attr.is_required + attr_dict["order"] = attr.order + attr_dict["default_show"] = attr.default_show + result.append(attr_dict) + return result + + @staticmethod + def _check(type_id, attr_ids): + CITypeManager.check_is_existed(type_id) + + if not attr_ids or not isinstance(attr_ids, list): + return abort(400, "Attributes are required") + + for attr_id in attr_ids: + AttributeCache.get(attr_id) or abort(404, "Attribute <{0}> is not existed".format(attr_id)) + + @classmethod + def add(cls, type_id, attr_ids=None, **kwargs): + """ + add attributes to CIType + :param type_id: + :param attr_ids: list + :param kwargs: + :return: + """ + cls._check(type_id, attr_ids) + + for attr_id in attr_ids: + existed = CITypeAttribute.get_by(type_id=type_id, + attr_id=attr_id, + first=True, + to_dict=False) + if existed is not None: + continue + + current_app.logger.debug(attr_id) + CITypeAttribute.create(type_id=type_id, attr_id=attr_id, **kwargs) + + CITypeAttributeCache.clean(type_id) + + @classmethod + def update(cls, type_id, attributes): + """ + update attributes to CIType + :param type_id: + :param attributes: list + :return: + """ + cls._check(type_id, [i.get('attr_id') for i in attributes]) + + for attr in attributes: + existed = CITypeAttribute.get_by(type_id=type_id, + attr_id=attr.get("attr_id"), + first=True, + to_dict=False) + if existed is None: + continue + + existed.update(**attr) + + CITypeAttributeCache.clean(type_id) + + @classmethod + def delete(cls, type_id, attr_ids=None): + """ + delete attributes from CIType + :param type_id: + :param attr_ids: list + :return: + """ + cls._check(type_id, attr_ids) + + for attr_id in attr_ids: + existed = CITypeAttribute.get_by(type_id=type_id, + attr_id=attr_id, + first=True, + to_dict=False) + if existed is not None: + existed.soft_delete() + + CITypeAttributeCache.clean(type_id) + + +class CITypeRelationManager(object): + """ + manage relation between CITypes + """ + + def __init__(self): + pass + + @staticmethod + def _wrap_relation_type_dict(type_id, relation_inst): + ci_type_dict = CITypeCache.get(type_id).to_dict() + ci_type_dict["ctr_id"] = relation_inst.id + ci_type_dict["attributes"] = CITypeAttributeManager.get_attributes_by_type_id(ci_type_dict["id"]) + ci_type_dict["relation_type"] = relation_inst.relation_type.name + return ci_type_dict + + @classmethod + def get_children(cls, parent_id): + children = CITypeRelation.get_by(parent_id=parent_id, to_dict=False) + + return [cls._wrap_relation_type_dict(child.child_id, child) for child in children] + + @classmethod + def get_parents(cls, child_id): + parents = CITypeRelation.get_by(child_id=child_id, to_dict=False) + + return [cls._wrap_relation_type_dict(parent.parent_id, parent) for parent in parents] + + @staticmethod + def _get(parent_id, child_id): + return CITypeRelation.get_by(parent_id=parent_id, + child_id=child_id, + to_dict=False, + first=True) + + @classmethod + def add(cls, parent, child, relation_type_id): + p = CITypeManager.check_is_existed(parent) + c = CITypeManager.check_is_existed(child) + + existed = cls._get(p.id, c.id) + if existed is not None: + existed.update(relation_type_id=relation_type_id) + else: + existed = CITypeRelation.create(parent_id=p.id, + child_id=c.id, + relation_type_id=relation_type_id) + return existed.id + + @staticmethod + def delete(_id): + ctr = CITypeRelation.get_by_id(_id) or abort(404, "Type relation <{0}> is not found".format(_id)) + ctr.soft_delete() + + @classmethod + def delete_2(cls, parent, child): + ctr = cls._get(parent, child) + return cls.delete(ctr.id) + + +class CITypeAttributeGroupManager(object): + @staticmethod + def get_by_type_id(type_id, need_other=None): + groups = CITypeAttributeGroup.get_by(type_id=type_id) + groups = sorted(groups, key=lambda x: x["order"]) + grouped = list() + for group in groups: + items = CITypeAttributeGroupItem.get_by(group_id=group["id"], to_dict=False) + items = sorted(items, key=lambda x: x.order) + group["attributes"] = [AttributeCache.get(i.attr_id).to_dict() for i in items] + grouped.extend([i.attr_id for i in items]) + + if need_other is not None: + grouped = set(grouped) + attributes = CITypeAttributeManager.get_attributes_by_type_id(type_id) + other_attributes = [attr for attr in attributes if attr["id"] not in grouped] + groups.append(dict(attributes=other_attributes)) + + return groups + + @staticmethod + def create_or_update(type_id, name, attr_order, group_order=0): + """ + create or update + :param type_id: + :param name: + :param group_order: group order + :param attr_order: + :return: + """ + existed = CITypeAttributeGroup.get_by(type_id=type_id, name=name, first=True, to_dict=False) \ + or CITypeAttributeGroup.create(type_id=type_id, name=name, order=group_order) + existed.update(order=group_order) + attr_order = dict(attr_order) + current_app.logger.info(attr_order) + existed_items = CITypeAttributeGroupItem.get_by(group_id=existed.id, to_dict=False) + for item in existed_items: + if item.attr_id not in attr_order: + item.soft_delete() + else: + item.update(order=attr_order[item.attr_id]) + + existed_items = {item.attr_id: 1 for item in existed_items} + for attr_id, order in attr_order.items(): + if attr_id not in existed_items: + CITypeAttributeGroupItem.create(group_id=existed.id, attr_id=attr_id, order=order) + + return existed + + @classmethod + def update(cls, group_id, name, attr_order, group_order=0): + group = CITypeAttributeGroup.get_by_id(group_id) or abort(404, "Group <{0}> does not exist".format(group_id)) + other = CITypeAttributeGroup.get_by(type_id=group.type_id, name=name, first=True, to_dict=False) + if other is not None and other.id != group.id: + return abort(400, "Group <{0}> duplicate".format(name)) + if name is not None: + group.update(name=name) + + cls.create_or_update(group.type_id, name, attr_order, group_order) + + @staticmethod + def delete(group_id): + group = CITypeAttributeGroup.get_by_id(group_id) \ + or abort(404, "AttributeGroup <{0}> does not exist".format(group_id)) + group.soft_delete() + + items = CITypeAttributeGroupItem.get_by(group_id=group_id, to_dict=False) + for item in items: + item.soft_delete() + + return group_id diff --git a/api/lib/cmdb/const.py b/api/lib/cmdb/const.py new file mode 100644 index 0000000..e5bfb55 --- /dev/null +++ b/api/lib/cmdb/const.py @@ -0,0 +1,148 @@ +# -*- coding:utf-8 -*- + +import datetime + +import six +from markupsafe import escape + +from api.models.cmdb import Attribute +from api.models.cmdb import TextChoice +from api.models.cmdb import FloatChoice +from api.models.cmdb import IntegerChoice +from api.models.cmdb import CIValueText +from api.models.cmdb import CIValueInteger +from api.models.cmdb import CIValueFloat +from api.models.cmdb import CIValueDateTime +from api.models.cmdb import CIIndexValueDateTime +from api.models.cmdb import CIIndexValueFloat +from api.models.cmdb import CIIndexValueInteger +from api.models.cmdb import CIIndexValueText +from api.lib.cmdb.cache import AttributeCache + + +def string2int(x): + return int(float(x)) + + +def str2datetime(x): + try: + return datetime.datetime.strptime(x, "%Y-%m-%d") + except ValueError: + pass + + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + + +type_map = { + 'deserialize': { + Attribute.INT: string2int, + Attribute.FLOAT: float, + Attribute.TEXT: escape, + Attribute.TIME: escape, + Attribute.DATETIME: str2datetime, + Attribute.DATE: str2datetime, + }, + 'serialize': { + Attribute.INT: int, + Attribute.FLOAT: float, + Attribute.TEXT: str, + Attribute.TIME: str, + Attribute.DATE: lambda x: x.strftime("%Y%m%d"), + Attribute.DATETIME: lambda x: x.strftime("%Y%m%d %H:%M:%S"), + }, + 'serialize2': { + Attribute.INT: int, + Attribute.FLOAT: float, + Attribute.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + Attribute.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + Attribute.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + Attribute.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + }, + 'choice': { + Attribute.INT: IntegerChoice, + Attribute.FLOAT: FloatChoice, + Attribute.TEXT: TextChoice, + }, + 'table': { + Attribute.INT: CIValueInteger, + Attribute.TEXT: CIValueText, + Attribute.DATETIME: CIValueDateTime, + Attribute.DATE: CIValueDateTime, + Attribute.TIME: CIValueText, + Attribute.FLOAT: CIValueFloat, + 'index_{0}'.format(Attribute.INT): CIIndexValueInteger, + 'index_{0}'.format(Attribute.TEXT): CIIndexValueText, + 'index_{0}'.format(Attribute.DATETIME): CIIndexValueDateTime, + 'index_{0}'.format(Attribute.DATE): CIIndexValueDateTime, + 'index_{0}'.format(Attribute.TIME): CIIndexValueText, + 'index_{0}'.format(Attribute.FLOAT): CIIndexValueFloat, + }, + 'table_name': { + Attribute.INT: 'c_value_integers', + Attribute.TEXT: 'c_value_texts', + Attribute.DATETIME: 'c_value_datetime', + Attribute.DATE: 'c_value_datetime', + Attribute.TIME: 'c_value_texts', + Attribute.FLOAT: 'c_value_floats', + 'index_{0}'.format(Attribute.INT): 'c_value_index_integers', + 'index_{0}'.format(Attribute.TEXT): 'c_value_index_texts', + 'index_{0}'.format(Attribute.DATETIME): 'c_value_index_datetime', + 'index_{0}'.format(Attribute.DATE): 'c_value_index_datetime', + 'index_{0}'.format(Attribute.TIME): 'c_value_index_texts', + 'index_{0}'.format(Attribute.FLOAT): 'c_value_index_floats', + } +} + + +class TableMap(object): + def __init__(self, attr_name=None): + self.attr_name = attr_name + + @property + def table(self): + attr = AttributeCache.get(self.attr_name) + i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type + return type_map["table"].get(i) + + @property + def table_name(self): + attr = AttributeCache.get(self.attr_name) + i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type + return type_map["table_name"].get(i) + + +class ExistPolicy(object): + REJECT = "reject" + NEED = "need" + IGNORE = "ignore" + REPLACE = "replace" + + +class OperateType(object): + ADD = "0" + DELETE = "1" + UPDATE = "2" + + +class RetKey(object): + ID = "id" + NAME = "name" + ALIAS = "alias" + + +class ResourceType(object): + CI = "CIType" + + +class PermEnum(object): + ADD = "add" + UPDATE = "update" + DELETE = "delete" + READ = "read" + + +class RoleEnum(object): + CONFIG = "admin" + +CMDB_QUEUE = "cmdb_async" +REDIS_PREFIX = "CMDB_CI" diff --git a/api/lib/cmdb/history.py b/api/lib/cmdb/history.py new file mode 100644 index 0000000..2116370 --- /dev/null +++ b/api/lib/cmdb/history.py @@ -0,0 +1,126 @@ +# -*- coding:utf-8 -*- + + +from flask import g +from flask import abort + + +from api.extensions import db +from api.models.cmdb import Attribute +from api.models.cmdb import OperationRecord +from api.models.cmdb import AttributeHistory +from api.models.cmdb import CIRelationHistory +from api.models.account import UserCache +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import RelationTypeCache + + +class AttributeHistoryManger(object): + + @staticmethod + def get_records(start, end, username, page, page_size): + records = db.session.query(OperationRecord).filter(OperationRecord.deleted.is_(False)) + numfound = db.session.query(db.func.count(OperationRecord.id)).filter(OperationRecord.deleted.is_(False)) + if start: + records = records.filter(OperationRecord.created_at >= start) + numfound = numfound.filter(OperationRecord.created_at >= start) + if end: + records = records.filter(OperationRecord.created_at <= end) + numfound = records.filter(OperationRecord.created_at <= end) + if username: + user = UserCache.get(username) + if user: + records = records.filter(OperationRecord.uid == user.uid) + else: + return abort(404, "User <{0}> is not found".format(username)) + + records = records.order_by(-OperationRecord.id).offset(page_size * (page - 1)).limit(page_size).all() + total = len(records) + numfound = numfound.first()[0] + res = [] + for record in records: + _res = record.to_dict() + _res["user"] = UserCache.get(_res.get("uid")).nickname or UserCache.get(_res.get("uid")).username + + attr_history = AttributeHistory.get_by(record_id=_res.get("id"), to_dict=False) + _res["attr_history"] = [AttributeCache.get(h.attr_id).attr_alias for h in attr_history] + + rel_history = CIRelationHistory.get_by(record_id=_res.get("id"), to_dict=False) + rel_statis = {} + for rel in rel_history: + if rel.operate_type not in rel_statis: + rel_statis[rel.operate_type] = 1 + else: + rel_statis[rel.operate_type] += 1 + _res["rel_history"] = rel_statis + res.append(_res) + + return numfound, total, res + + @staticmethod + def get_by_ci_id(ci_id): + res = db.session.query(AttributeHistory, Attribute, OperationRecord).join( + Attribute, Attribute.id == AttributeHistory.attr_id).join( + OperationRecord, OperationRecord.id == AttributeHistory.record_id).filter( + AttributeHistory.ci_id == ci_id).order_by(OperationRecord.id.desc()) + return [dict(attr_name=i.Attribute.name, + attr_alias=i.Attribute.alias, + operate_type=i.AttributeHistory.operate_type, + username=UserCache.get(i.OperationRecord.uid).nickname, + old=i.AttributeHistory.old, + new=i.AttributeHistory.new, + created_at=i.OperationRecord.created_at.strftime('%Y-%m-%d %H:%M:%S'), + record_id=i.OperationRecord.id, + hid=i.AttributeHistory.id + ) for i in res] + + @staticmethod + def get_record_detail(record_id): + from api.lib.cmdb.ci import CIManager + + record = OperationRecord.get_by_id(record_id) or abort(404, "Record <{0}> is not found".format(record_id)) + + username = UserCache.get(record.uid).nickname or UserCache.get(record.uid).username + timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S") + attr_history = AttributeHistory.get_By(record_id=record_id, to_dict=False) + rel_history = CIRelationHistory.get_by(record_id=record_id, to_dict=False) + + attr_dict, rel_dict = dict(), {"add": [], "delete": []} + for attr_h in attr_history: + attr_dict[AttributeCache.get(attr_h.attr_id).alias] = dict( + old=attr_h.old, + new=attr_h.new, + operate_type=attr_h.operate_type) + + for rel_h in rel_history: + first = CIManager.get_ci_by_id(rel_h.first_ci_id) + second = CIManager.get_ci_by_id(rel_h.second_ci_id) + rel_dict[rel_h.operate_type].append((first, RelationTypeCache.get(rel_h.relation_type_id).name, second)) + + return username, timestamp, attr_dict, rel_dict + + @staticmethod + def add(ci_id, history_list): + record = OperationRecord.create(uid=g.user.uid) + + for attr_id, operate_type, old, new in history_list or []: + AttributeHistory.create(attr_id=attr_id, + operate_type=operate_type, + old=old, + new=new, + ci_id=ci_id, + record_id=record.id) + + +class CIRelationHistoryManager(object): + + @staticmethod + def add(rel_obj, operate_type=CIRelationHistory.ADD): + record = OperationRecord.create(uid=g.user.uid) + + CIRelationHistory.create(relation_id=rel_obj.id, + record_id=record.id, + operate_type=operate_type, + first_ci_id=rel_obj.first_ci_id, + second_ci_id=rel_obj.second_ci_id, + relation_type_id=rel_obj.relation_type_id) diff --git a/api/lib/cmdb/preference.py b/api/lib/cmdb/preference.py new file mode 100644 index 0000000..0fabe64 --- /dev/null +++ b/api/lib/cmdb/preference.py @@ -0,0 +1,137 @@ +# -*- coding:utf-8 -*- + +import six +import toposort +from flask import g +from flask import abort + +from api.extensions import db +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.cache import CITypeAttributeCache +from api.models.cmdb import PreferenceShowAttributes +from api.models.cmdb import PreferenceTreeView +from api.models.cmdb import PreferenceRelationView +from api.models.cmdb import CITypeAttribute +from api.lib.cmdb.attribute import AttributeManager + + +class PreferenceManager(object): + + @staticmethod + def get_types(instance=False, tree=False): + types = db.session.query(PreferenceShowAttributes.type_id).filter( + PreferenceShowAttributes.uid == g.user.uid).filter( + PreferenceShowAttributes.deleted.is_(False)).group_by(PreferenceShowAttributes.type_id).all() \ + if instance else [] + tree_types = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=False) if tree else [] + type_ids = list(set([i.type_id for i in types + tree_types])) + return [CITypeCache.get(type_id).to_dict() for type_id in type_ids] + + @staticmethod + def get_show_attributes(type_id): + if not isinstance(type_id, six.integer_types): + type_id = CITypeCache.get(type_id).id + + attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join( + CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter( + PreferenceShowAttributes.uid == g.user.uid).filter( + PreferenceShowAttributes.type_id == type_id).filter( + PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).filter( + CITypeAttribute.type_id == type_id).order_by( + CITypeAttribute.order).all() + result = [i.PreferenceShowAttributes.attr.to_dict() for i in attrs] + is_subscribed = True + if not attrs: + attrs = db.session.query(CITypeAttribute).filter( + CITypeAttribute.type_id == type_id).filter( + CITypeAttribute.deleted.is_(False)).filter( + CITypeAttribute.default_show.is_(True)).order_by(CITypeAttribute.order) + result = [i.attr.to_dict() for i in attrs] + is_subscribed = False + + for i in result: + if i["is_choice"]: + i.update(dict(choice_value=AttributeManager.get_choice_values(i["id"], i["value_type"]))) + + return is_subscribed, result + + @classmethod + def create_or_update_show_attributes(cls, type_id, attr_order): + existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=g.user.uid, to_dict=False) + for _attr, order in attr_order: + attr = AttributeCache.get(_attr) or abort(404, "Attribute <{0}> does not exist".format(_attr)) + existed = PreferenceShowAttributes.get_by(type_id=type_id, + uid=g.user.uid, + attr_id=attr.id, + first=True, + to_dict=False) + if existed is None: + PreferenceShowAttributes.create(type_id=type_id, + uid=g.user.uid, + attr_id=attr.id, + order=order) + else: + existed.update(order=order) + + attr_dict = {int(i): j for i, j in attr_order} + for i in existed_all: + if i.attr_id not in attr_dict: + i.soft_delete() + + @staticmethod + def get_tree_view(): + res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True) + for item in res: + if item["levels"]: + item.update(CITypeCache.get(item['type_id']).to_dict()) + item.update(dict(levels=[AttributeCache.get(l).to_dict() + for l in item["levels"].split(",") if AttributeCache.get(l)])) + + return res + + @staticmethod + def create_or_update_tree_view(type_id, levels): + attrs = CITypeAttributeCache.get(type_id) + for idx, i in enumerate(levels): + for attr in attrs: + attr = AttributeCache.get(attr.attr_id) + if i == attr.id or i == attr.name or i == attr.alias: + levels[idx] = str(attr.id) + levels = ",".join(levels) + + existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True) + if existed is not None: + if not levels: + existed.soft_delete() + return existed + return existed.update(levels=levels) + elif levels: + return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid) + + @staticmethod + def get_relation_view(): + views = PreferenceRelationView.get_by(to_dict=True) + result = dict() + for view in views: + result.setdefault(view['name'], []).append(view) + + for view_name in result: + result[view_name] = toposort.toposort_flatten({i['child_id']: {i['parent_id']} for i in result[view_name]}) + + return result + + @staticmethod + def create_or_update_relation_view(name, parent_id, child_id): + existed = PreferenceRelationView.get_by(name=name, parent_id=parent_id, child_id=child_id, + to_dict=False, first=True) + if existed is None: + return PreferenceRelationView.create(name=name, parent_id=parent_id, child_id=child_id) + + return existed + + @staticmethod + def delete_relation_view(name): + for existed in PreferenceRelationView.get_by(name=name, to_dict=False): + existed.soft_delete() + return name diff --git a/api/lib/cmdb/query_sql.py b/api/lib/cmdb/query_sql.py new file mode 100644 index 0000000..c84fd64 --- /dev/null +++ b/api/lib/cmdb/query_sql.py @@ -0,0 +1,63 @@ +# -*- coding:utf-8 -*- + + +QUERY_CIS_BY_VALUE_TABLE = """ + SELECT attr.name AS attr_name, + attr.alias AS attr_alias, + attr.value_type, + attr.is_list, + c_cis.type_id, + {0}.ci_id, + {0}.attr_id, + {0}.value + FROM {0} + INNER JOIN c_cis ON {0}.ci_id=c_cis.id + AND {0}.`ci_id` IN ({1}) + INNER JOIN c_attributes as attr ON attr.id = {0}.attr_id +""" + +# {2}: value_table +QUERY_CIS_BY_IDS = """ + SELECT A.ci_id, + A.type_id, + A.attr_id, + A.attr_name, + A.attr_alias, + A.value, + A.value_type, + A.is_list + FROM + ({1}) AS A {0} + ORDER BY A.ci_id; +""" + +FACET_QUERY1 = """ + SELECT {0}.value, + count({0}.ci_id) + FROM {0} + INNER JOIN c_attributes AS attr ON attr.id={0}.attr_id + WHERE attr.name="{1}" + GROUP BY {0}.ci_id; +""" + +FACET_QUERY = """ + SELECT {0}.value, + count({0}.ci_id) + FROM {0} + INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id + WHERE {0}.attr_id={2:d} + GROUP BY {0}.value +""" + +QUERY_CI_BY_ATTR_NAME = """ + SELECT {0}.ci_id + FROM {0} + WHERE {0}.attr_id={1:d} + AND {0}.value {2} +""" + +QUERY_CI_BY_TYPE = """ + SELECT c_cis.id AS ci_id + FROM c_cis + WHERE c_cis.type_id in ({0}) +""" diff --git a/api/lib/cmdb/relation_type.py b/api/lib/cmdb/relation_type.py new file mode 100644 index 0000000..e932510 --- /dev/null +++ b/api/lib/cmdb/relation_type.py @@ -0,0 +1,37 @@ +# -*- coding:utf-8 -*- + + +from flask import abort + +from api.models.cmdb import RelationType + + +class RelationTypeManager(object): + @staticmethod + def get_all(): + return RelationType.get_by(to_dict=False) + + @classmethod + def get_names(cls): + return [i.name for i in cls.get_all()] + + @classmethod + def get_pairs(cls): + return [(i.id, i.name) for i in cls.get_all()] + + @staticmethod + def add(name): + RelationType.get_by(name=name, first=True, to_dict=False) and abort(400, "It's already existed") + return RelationType.create(name=name) + + @staticmethod + def update(rel_id, name): + existed = RelationType.get_by_id(rel_id) or abort(404, "RelationType <{0}> does not exist".format(rel_id)) + + return existed.update(name=name) + + @staticmethod + def delete(rel_id): + existed = RelationType.get_by_id(rel_id) or abort(404, "RelationType <{0}> does not exist".format(rel_id)) + + existed.soft_delete() diff --git a/api/lib/cmdb/search.py b/api/lib/cmdb/search.py new file mode 100644 index 0000000..95de15d --- /dev/null +++ b/api/lib/cmdb/search.py @@ -0,0 +1,356 @@ +# -*- coding:utf-8 -*- + + +import time + +from flask import current_app + +from api.extensions import db +from api.lib.utils import handle_arg_list +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.ci import CIManager +from api.lib.cmdb.const import RetKey +from api.lib.cmdb.const import TableMap +from api.lib.cmdb.query_sql import FACET_QUERY +from api.lib.cmdb.query_sql import QUERY_CI_BY_ATTR_NAME +from api.lib.cmdb.query_sql import QUERY_CI_BY_TYPE +from api.models.cmdb import Attribute +from api.models.cmdb import CI + + +class SearchError(Exception): + def __init__(self, v): + self.v = v + + def __str__(self): + return self.v + + +class Search(object): + def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None): + self.orig_query = query + self.fl = fl + self.facet_field = facet_field + self.page = page + self.ret_key = ret_key + self.count = count + self.sort = sort + self.query_sql = "" + self.type_id_list = [] + self.only_type_query = False + + @staticmethod + def _operator_proc(key): + operator = "&" + if key.startswith("+"): + key = key[1:].strip() + elif key.startswith("-"): + operator = "|" + key = key[1:].strip() + elif key.startswith("~"): + operator = "~" + key = key[1:].strip() + + return operator, key + + def _attr_name_proc(self, key): + operator, key = self._operator_proc(key) + + if key in ('ci_type', 'type', '_type'): + return '_type', Attribute.TEXT, operator, None + + if key in ('id', 'ci_id', '_id'): + return '_id', Attribute.TEXT, operator, None + + attr = AttributeCache.get(key) + if attr: + return attr.name, attr.value_type, operator, attr + else: + raise SearchError("{0} is not existed".format(key)) + + def _type_query_handler(self, v): + new_v = v[1:-1].split(";") if v.startswith("(") and v.endswith(")") else [v] + for _v in new_v: + ci_type = CITypeCache.get(_v) + if ci_type is not None: + self.type_id_list.append(str(ci_type.id)) + + if self.type_id_list: + type_ids = ",".join(self.type_id_list) + _query_sql = QUERY_CI_BY_TYPE.format(type_ids) + if self.only_type_query: + return _query_sql + else: + return "" + return "" + + @staticmethod + def _in_query_handler(attr, v): + new_v = v[1:-1].split(";") + table_name = TableMap(attr_name=attr.name).table_name + in_query = " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format(_v.replace("*", "%")) for _v in new_v]) + _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, in_query) + return _query_sql + + @staticmethod + def _range_query_handler(attr, v): + start, end = [x.strip() for x in v[1:-1].split("_TO_")] + table_name = TableMap(attr_name=attr.name).table_name + range_query = "BETWEEN '{0}' AND '{1}'".format(start.replace("*", "%"), end.replace("*", "%")) + _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, range_query) + return _query_sql + + @staticmethod + def _comparison_query_handler(attr, v): + table_name = TableMap(attr_name=attr.name).table_name + if v.startswith(">=") or v.startswith("<="): + comparison_query = "{0} '{1}'".format(v[:2], v[2:].replace("*", "%")) + else: + comparison_query = "{0} '{1}'".format(v[0], v[1:].replace("*", "%")) + _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, comparison_query) + return _query_sql + + @staticmethod + def __sort_by(field): + field = field or "" + sort_type = "ASC" + if field.startswith("+"): + field = field[1:] + elif field.startswith("-"): + field = field[1:] + sort_type = "DESC" + return field, sort_type + + def __sort_by_id(self, sort_type, query_sql): + ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}" + + if self.only_type_query: + return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( + (self.page - 1) * self.count, sort_type, self.count)) + + elif self.type_id_list: + self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format( + query_sql, + "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id in ({0}) ".format( + ",".join(self.type_id_list))) + + return ret_sql.format( + query_sql, + "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id in ({3}) " + "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( + (self.page - 1) * self.count, sort_type, self.count, ",".join(self.type_id_list))) + + else: + self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format( + query_sql, + "INNER JOIN c_cis on c_cis.id=B.ci_id ") + + return ret_sql.format( + query_sql, + "INNER JOIN c_cis on c_cis.id=B.ci_id " + "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format((self.page - 1) * self.count, sort_type, self.count)) + + def __sort_by_field(self, field, sort_type, query_sql): + attr = AttributeCache.get(field) + attr_id = attr.id + + table_name = TableMap(attr_name=attr.name).table_name + _v_query_sql = """SELECT {0}.ci_id, {1}.value + FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id + WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id) + new_table = _v_query_sql + + if self.only_type_query or not self.type_id_list: + return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value " \ + "FROM ({0}) AS C " \ + "ORDER BY C.value {2} " \ + "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count) + + elif self.type_id_list: + self.query_sql = """SELECT C.ci_id + FROM ({0}) AS C + INNER JOIN cis on c_cis.id=C.ci_id + WHERE cis.type_id in ({1})""".format(new_table, ",".join(self.type_id_list)) + + return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value + FROM ({0}) AS C + INNER JOIN cis on c_cis.id=C.ci_id + WHERE cis.type_id in ({4}) + ORDER BY C.value {2} + LIMIT {1:d}, {3};""".format(new_table, + (self.page - 1) * self.count, + sort_type, self.count, + ",".join(self.type_id_list)) + + def _sort_query_handler(self, field, query_sql): + + field, sort_type = self.__sort_by(field) + + if field in ("_id", "ci_id") or not field: + return self.__sort_by_id(sort_type, query_sql) + else: + return self.__sort_by_field(field, sort_type, query_sql) + + @staticmethod + def _wrap_sql(operator, alias, _query_sql, query_sql): + if operator == "&": + query_sql = """SELECT * FROM ({0}) as {1} + INNER JOIN ({2}) as {3} USING(ci_id)""".format(query_sql, alias, _query_sql, alias + "A") + + elif operator == "|": + query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql) + + elif operator == "~": + query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id) + WHERE {3}.ci_id is NULL""".format(query_sql, alias, _query_sql, alias + "A") + + return query_sql + + def _execute_sql(self, query_sql): + v_query_sql = self._sort_query_handler(self.sort, query_sql) + + start = time.time() + execute = db.session.execute + current_app.logger.debug(v_query_sql) + res = execute(v_query_sql).fetchall() + end_time = time.time() + current_app.logger.debug("query ci ids time is: {0}".format(end_time - start)) + + numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0] + current_app.logger.debug("statistics ci ids time is: {0}".format(time.time() - end_time)) + + return numfound, res + + def __confirm_type_first(self, queries): + for q in queries: + if q.startswith("_type"): + queries.remove(q) + queries.insert(0, q) + if len(queries) == 1 or queries[1].startswith("-") or queries[1].startswith("~"): + self.only_type_query = True + return queries + + def __query_build_by_field(self, queries): + query_sql, alias, operator = "", "A", "&" + is_first, only_type_query_special = True, True + + for q in queries: + _query_sql = "" + if ":" in q: + k = q.split(":")[0].strip() + v = ":".join(q.split(":")[1:]).strip() + current_app.logger.debug(v) + field, field_type, operator, attr = self._attr_name_proc(k) + if field == "_type": + _query_sql = self._type_query_handler(v) + current_app.logger.debug(_query_sql) + elif field == "_id": # exclude all others + ci = CI.get_by_id(v) + if ci is not None: + return 1, [str(v)] + elif field: + if attr is None: + raise SearchError("{0} is not found".format(field)) + + # in query + if v.startswith("(") and v.endswith(")"): + _query_sql = self._in_query_handler(attr, v) + # range query + elif v.startswith("[") and v.endswith("]") and "_TO_" in v: + _query_sql = self._range_query_handler(attr, v) + # comparison query + elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"): + _query_sql = self._comparison_query_handler(attr, v) + else: + table_name = TableMap(attr_name=attr.name).table_name + _query_sql = QUERY_CI_BY_ATTR_NAME.format( + table_name, attr.id, 'LIKE "{0}"'.format(v.replace("*", "%"))) + else: + raise SearchError("argument q format invalid: {0}".format(q)) + elif q: + raise SearchError("argument q format invalid: {0}".format(q)) + + if is_first and _query_sql and not self.only_type_query: + query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, alias) + is_first = False + alias += "A" + elif self.only_type_query and only_type_query_special: + is_first = False + only_type_query_special = False + query_sql = _query_sql + elif _query_sql: + query_sql = self._wrap_sql(operator, alias, _query_sql, query_sql) + alias += "AA" + return None, query_sql + + def _query_build_raw(self): + + queries = handle_arg_list(self.orig_query) + queries = self.__confirm_type_first(queries) + current_app.logger.debug(queries) + + ret, query_sql = self.__query_build_by_field(queries) + if ret is not None: + return ret, query_sql + + s = time.time() + if query_sql: + self.query_sql = query_sql + current_app.logger.debug(query_sql) + numfound, res = self._execute_sql(query_sql) + current_app.logger.info("query ci ids is: {0}".format(time.time() - s)) + return numfound, [_res[0] for _res in res] + + return 0, [] + + def _facet_build(self): + facet = {} + for f in self.facet_field: + k, field_type, _, attr = self._attr_name_proc(f) + if k: + table_name = TableMap(attr_name=k).table_name + query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id) + current_app.logger.debug(query_sql) + result = db.session.execute(query_sql).fetchall() + facet[k] = result + + facet_result = dict() + for k, v in facet.items(): + if not k.startswith('_'): + a = getattr(AttributeCache.get(k), self.ret_key) + facet_result[a] = [(f[0], f[1], a) for f in v] + + return facet_result + + def _fl_build(self): + _fl = list() + for f in self.fl: + k, _, _, _ = self._attr_name_proc(f) + if k: + _fl.append(k) + + return _fl + + def search(self): + numfound, ci_ids = self._query_build_raw() + ci_ids = list(map(str, ci_ids)) + + _fl = self._fl_build() + + if self.facet_field and numfound: + facet = self._facet_build() + else: + facet = dict() + + response, counter = [], {} + if ci_ids: + response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl) + for res in response: + ci_type = res.get("ci_type") + if ci_type not in counter.keys(): + counter[ci_type] = 0 + counter[ci_type] += 1 + total = len(response) + + return response, counter, total, self.page, numfound, facet diff --git a/api/lib/cmdb/value.py b/api/lib/cmdb/value.py new file mode 100644 index 0000000..951b29e --- /dev/null +++ b/api/lib/cmdb/value.py @@ -0,0 +1,138 @@ +# -*- coding:utf-8 -*- + + +import markupsafe + +from flask import abort + +from api.extensions import db +from api.lib.utils import handle_arg_list +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.attribute import AttributeManager +from api.lib.cmdb.const import type_map +from api.lib.cmdb.const import TableMap +from api.lib.cmdb.const import ExistPolicy +from api.lib.cmdb.const import OperateType +from api.lib.cmdb.history import AttributeHistoryManger + + +class AttributeValueManager(object): + """ + manage CI attribute values + """ + + def __init__(self): + pass + + @staticmethod + def _get_attr(key): + """ + :param key: id, name or alias + :return: attribute instance + """ + return AttributeCache.get(key) + + def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False): + """ + + :param fields: + :param ci_id: + :param ret_key: It can be name or alias + :param unique_key: primary attribute + :param use_master: Only for master-slave read-write separation + :return: + """ + res = dict() + for field in fields: + attr = self._get_attr(field) + if not attr: + continue + value_table = TableMap(attr_name=attr.name).table + rs = value_table.get_by(ci_id=ci_id, + attr_id=attr.id, + use_master=use_master, + to_dict=False) + field_name = getattr(attr, ret_key) + + if attr.is_list: + res[field_name] = [type_map["serialize"][attr.value_type](i.value) for i in rs] + else: + res[field_name] = type_map["serialize"][attr.value_type](rs[0].value) if rs else None + + if unique_key is not None and attr.id == unique_key.id and rs: + res['unique'] = unique_key.name + + return res + + @staticmethod + def __deserialize_value(value_type, value): + if not value: + return value + deserialize = type_map["deserialize"][value_type] + try: + v = deserialize(value) + if isinstance(v, markupsafe.Markup): + v = str(v) + return v + except ValueError: + return abort(400, "attribute value <{0}> is invalid".format(value)) + + @staticmethod + def __check_is_choice(attr_id, value_type, value): + choice_values = AttributeManager.get_choice_values(attr_id, value_type) + if value not in choice_values: + return abort(400, "{0} does not existed in choice values".format(value)) + + @staticmethod + def __check_is_unique(value_table, attr_id, ci_id, value): + db.session.query(value_table.attr_id).filter( + value_table.attr_id == attr_id).filter(value_table.deleted.is_(False)).filter( + value_table.value == value).filter(value_table.ci_id != ci_id).first() \ + and abort(400, "attribute <{0}> value {1} must be unique".format(attr_id, value)) + + def _validate(self, attr, value, value_table, ci_id): + v = self.__deserialize_value(attr.value_type, value) + + attr.is_choice and value and self.__check_is_choice(attr.id, attr.value_type, v) + attr.is_unique and self.__check_is_unique(value_table, attr.id, ci_id, v) + + return v + + @staticmethod + def _write_change(ci_id, attr_id, operate_type, old, new): + AttributeHistoryManger.add(ci_id, [(attr_id, operate_type, old, new)]) + + def create_or_update_attr_value(self, key, value, ci_id, _no_attribute_policy=ExistPolicy.IGNORE): + """ + add or update attribute value, then write history + :param key: id, name or alias + :param value: + :param ci_id: + :param _no_attribute_policy: ignore or reject + :return: + """ + attr = self._get_attr(key) + if attr is None: + if _no_attribute_policy == ExistPolicy.IGNORE: + return + if _no_attribute_policy == ExistPolicy.REJECT: + return abort(400, 'attribute {0} does not exist'.format(key)) + + value_table = TableMap(attr_name=attr.name).table + existed_attr = value_table.get_by(attr_id=attr.id, + ci_id=ci_id, + first=True, + to_dict=False) + existed_value = existed_attr and existed_attr.value + operate_type = OperateType.ADD if existed_attr is None else OperateType.UPDATE + + value_list = handle_arg_list(value) if attr.is_list else [value] + + for v in value_list: + v = self._validate(attr, v, value_table, ci_id) + if operate_type == OperateType.ADD: + value_table.create(ci_id=ci_id, attr_id=attr.id, value=v) + self._write_change(ci_id, attr.id, operate_type, None, v) + elif existed_attr.value != v: + existed_attr.update(value=v) + self._write_change(ci_id, attr.id, operate_type, existed_value, v) diff --git a/api/lib/database.py b/api/lib/database.py new file mode 100644 index 0000000..2ac9867 --- /dev/null +++ b/api/lib/database.py @@ -0,0 +1,121 @@ +# -*- coding:utf-8 -*- + +import datetime + +import six + +from api.extensions import db +from api.lib.exception import CommitException + + +class FormatMixin(object): + def to_dict(self): + return dict([(k.name, getattr(self, k.name)) for k in getattr(self, "__table__").columns]) + + @classmethod + def get_columns(cls): + return {k.name: 1 for k in getattr(cls, "__mapper__").c.values()} + + +class CRUDMixin(FormatMixin): + + @classmethod + def create(cls, flush=False, **kwargs): + return cls(**kwargs).save(flush=flush) + + def update(self, flush=False, **kwargs): + kwargs.pop("id", None) + for attr, value in six.iteritems(kwargs): + if value is not None: + setattr(self, attr, value) + if flush: + return self.save(flush=flush) + return self.save() + + def save(self, commit=True, flush=False): + db.session.add(self) + try: + if flush: + db.session.flush() + elif commit: + db.session.commit() + except Exception as e: + db.session.rollback() + raise CommitException(str(e)) + + return self + + def delete(self, flush=False): + db.session.delete(self) + try: + if flush: + return db.session.flush() + return db.session.commit() + except Exception as e: + db.session.rollback() + raise CommitException(str(e)) + + def soft_delete(self, flush=False): + setattr(self, "deleted", True) + setattr(self, "deleted_at", datetime.datetime.now()) + self.save(flush=flush) + + @classmethod + def get_by_id(cls, _id): + if any((isinstance(_id, six.string_types) and _id.isdigit(), + isinstance(_id, (int, float))), ): + return getattr(cls, "query").get(int(_id)) or None + + @classmethod + def get_by(cls, first=False, to_dict=True, fl=None, exclude=None, deleted=False, use_master=False, **kwargs): + db_session = db.session if not use_master else db.session().using_bind("master") + fl = fl.strip().split(",") if fl and isinstance(fl, six.string_types) else (fl or []) + exclude = exclude.strip().split(",") if exclude and isinstance(exclude, six.string_types) else (exclude or []) + + keys = cls.get_columns() + fl = [k for k in fl if k in keys] + fl = [k for k in keys if k not in exclude and not k.isupper()] if exclude else fl + fl = list(filter(lambda x: "." not in x, fl)) + + if hasattr(cls, "deleted") and deleted is not None: + kwargs["deleted"] = deleted + + if fl: + query = db_session.query(*[getattr(cls, k) for k in fl]) + query = query.filter_by(**kwargs) + result = [{k: getattr(i, k) for k in fl} for i in query] + else: + result = [i.to_dict() if to_dict else i for i in getattr(cls, 'query').filter_by(**kwargs)] + + return result[0] if first and result else (None if first else result) + + @classmethod + def get_by_like(cls, to_dict=True, **kwargs): + query = db.session.query(cls) + for k, v in kwargs.items(): + query = query.filter(getattr(cls, k).ilike('%{0}%'.format(v))) + return [i.to_dict() if to_dict else i for i in query] + + +class SoftDeleteMixin(object): + deleted_at = db.Column(db.DateTime) + deleted = db.Column(db.Boolean, index=True, default=False) + + +class TimestampMixin(object): + created_at = db.Column(db.DateTime, default=lambda: datetime.datetime.now()) + updated_at = db.Column(db.DateTime, onupdate=lambda: datetime.datetime.now()) + + +class SurrogatePK(object): + __table_args__ = {"extend_existing": True} + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + + +class Model(SoftDeleteMixin, TimestampMixin, CRUDMixin, db.Model, SurrogatePK): + __abstract__ = True + + +class CRUDModel(db.Model, CRUDMixin): + __abstract__ = True diff --git a/api/lib/decorator.py b/api/lib/decorator.py new file mode 100644 index 0000000..d4c6342 --- /dev/null +++ b/api/lib/decorator.py @@ -0,0 +1,35 @@ +# -*- coding:utf-8 -*- + + +from functools import wraps + +from flask import abort +from flask import request + + +def kwargs_required(*required_args): + def decorate(func): + @wraps(func) + def wrapper(*args, **kwargs): + for arg in required_args: + if arg not in kwargs: + return abort(400, "Argument <{0}> is required".format(arg)) + return func(*args, **kwargs) + + return wrapper + + return decorate + + +def args_required(*required_args): + def decorate(func): + @wraps(func) + def wrapper(*args, **kwargs): + for arg in required_args: + if arg not in request.values: + return abort(400, "Argument <{0}> is required".format(arg)) + return func(*args, **kwargs) + + return wrapper + + return decorate diff --git a/api/lib/exception.py b/api/lib/exception.py new file mode 100644 index 0000000..0a68289 --- /dev/null +++ b/api/lib/exception.py @@ -0,0 +1,5 @@ +# -*- coding:utf-8 -*- + + +class CommitException(Exception): + pass diff --git a/api/lib/http_cli.py b/api/lib/http_cli.py new file mode 100644 index 0000000..6c59777 --- /dev/null +++ b/api/lib/http_cli.py @@ -0,0 +1,48 @@ +# -*- coding:utf-8 -*- + + +import hashlib + +import requests +from future.moves.urllib.parse import urlparse +from flask import abort +from flask import g +from flask import current_app + + +def build_api_key(path, params): + g.user is not None or abort(403, u"您得登陆才能进行该操作") + key = g.user.key + secret = g.user.secret + values = "".join([str(params[k]) for k in sorted(params.keys()) + if params[k] is not None]) if params.keys() else "" + _secret = "".join([path, secret, values]).encode("utf-8") + params["_secret"] = hashlib.sha1(_secret).hexdigest() + params["_key"] = key + return params + + +def api_request(url, method="get", params=None, ret_key=None): + params = params or {} + resp = None + try: + method = method.lower() + params = build_api_key(urlparse(url).path, params) + if method == "get": + resp = getattr(requests, method)(url, params=params) + else: + resp = getattr(requests, method)(url, data=params) + if resp.status_code != 200: + return abort(resp.status_code, resp.json().get("message")) + resp = resp.json() + if ret_key is not None: + return resp.get(ret_key) + return resp + except Exception as e: + code = e.code if hasattr(e, "code") else None + if isinstance(code, int) and resp is not None: + return abort(code, resp.json().get("message")) + current_app.logger.warning(url) + current_app.logger.warning(params) + current_app.logger.error(str(e)) + return abort(500, "server unknown error") diff --git a/api/lib/mail.py b/api/lib/mail.py new file mode 100644 index 0000000..0fbf25d --- /dev/null +++ b/api/lib/mail.py @@ -0,0 +1,49 @@ +# -*- coding:utf-8 -*- + + +from flask import current_app + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.header import Header +from email.mime.image import MIMEImage +import smtplib +import time +from email import Utils + + +def send_mail(sender, receiver, subject, content, ctype="html", pics=()): + """subject and body are unicode objects""" + if not sender: + sender = current_app.config.get("DEFAULT_MAIL_SENDER") + smtpserver = current_app.config.get("MAIL_SERVER") + if ctype == "html": + msg = MIMEText(content, 'html', 'utf-8') + else: + msg = MIMEText(content, 'plain', 'utf-8') + + if len(pics) != 0: + msgRoot = MIMEMultipart('related') + msgText = MIMEText(content, 'html', 'utf-8') + msgRoot.attach(msgText) + i = 1 + for pic in pics: + fp = open(pic, "rb") + image = MIMEImage(fp.read()) + fp.close() + image.add_header('Content-ID', '' % i) + msgRoot.attach(image) + i += 1 + msg = msgRoot + + msg['Subject'] = Header(subject, 'utf-8') + msg['From'] = sender + msg['To'] = ';'.join(receiver) + msg['Message-ID'] = Utils.make_msgid() + msg['date'] = time.strftime('%a, %d %b %Y %H:%M:%S %z') + + smtp = smtplib.SMTP() + smtp.connect(smtpserver, 25) + # smtp.login(username, password) + smtp.sendmail(sender, receiver, msg.as_string()) + smtp.quit() diff --git a/api/lib/perm/__init__.py b/api/lib/perm/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/api/lib/perm/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/api/lib/perm/acl.py b/api/lib/perm/acl.py new file mode 100644 index 0000000..7902b04 --- /dev/null +++ b/api/lib/perm/acl.py @@ -0,0 +1,139 @@ +# -*- coding:utf-8 -*- + +import functools + +import six + +from flask import current_app, g, request +from flask import session, abort + +from api.extensions import cache + + +def get_access_token(): + return + + +class AccessTokenCache(object): + @classmethod + def get(cls): + if cache.get("AccessToken") is not None: + return cache.get("AccessToken") + + res = get_access_token() or "" + + cache.set("AccessToken", res, timeout=60 * 60) + return res + + @classmethod + def clean(cls): + cache.clear("AccessToken") + + +class ACLManager(object): + def __init__(self): + self.access_token = AccessTokenCache.get() + self.acl_session = dict(uid=session.get("uid"), + token=self.access_token) + + self.user_info = session["acl"] if "acl" in session else {} + + def add_resource(self, name, resource_type_name=None): + pass + + def grant_resource_to_role(self, name, role, resource_type_name=None): + pass + + def del_resource(self, name, resource_type_name=None): + pass + + def get_user_info(self, username): + return dict() + + def get_resources(self, resource_type_name=None): + if "acl" not in session: + abort(405) + return [] + + def has_permission(self, resource_name, resource_type, perm): + if "acl" not in session: + abort(405) + return True + + +def validate_permission(resources, resource_type, perm): + if not resources: + return + + if current_app.config.get("USE_ACL"): + if g.user.username == "worker": + return + + resources = [resources] if isinstance(resources, six.string_types) else resources + for resource in resources: + if not ACLManager().has_permission(resource, resource_type, perm): + return abort(403, "has no permission") + + +def can_access_resources(resource_type): + def decorator_can_access_resources(func): + @functools.wraps(func) + def wrapper_can_access_resources(*args, **kwargs): + if current_app.config.get("USE_ACL"): + res = ACLManager().get_resources(resource_type) + result = {i.get("name"): i.get("permissions") for i in res} + if hasattr(g, "resources"): + g.resources.update({resource_type: result}) + else: + g.resources = {resource_type: result} + return func(*args, **kwargs) + return wrapper_can_access_resources + return decorator_can_access_resources + + +def has_perm(resources, resource_type, perm): + def decorator_has_perm(func): + @functools.wraps(func) + def wrapper_has_perm(*args, **kwargs): + if not resources: + return + + if current_app.config.get("USE_ACL"): + validate_permission(resources, resource_type, perm) + + return func(*args, **kwargs) + return wrapper_has_perm + return decorator_has_perm + + +def has_perm_from_args(arg_name, resource_type, perm, callback=None): + def decorator_has_perm(func): + @functools.wraps(func) + def wrapper_has_perm(*args, **kwargs): + if not arg_name: + return + resource = request.view_args.get(arg_name) or request.values.get(arg_name) + if callback is not None and resource: + resource = callback(resource) + + if current_app.config.get("USE_ACL") and resource: + validate_permission(resource, resource_type, perm) + + return func(*args, **kwargs) + return wrapper_has_perm + return decorator_has_perm + + +def role_required(role_name): + def decorator_role_required(func): + @functools.wraps(func) + def wrapper_role_required(*args, **kwargs): + if not role_name: + return + + if current_app.config.get("USE_ACL"): + if role_name not in session.get("acl", {}).get("parentRoles", []): + return abort(403, "Role {0} is required".format(role_name)) + return func(*args, **kwargs) + return wrapper_role_required + return decorator_role_required diff --git a/api/lib/perm/auth.py b/api/lib/perm/auth.py new file mode 100644 index 0000000..e1637f4 --- /dev/null +++ b/api/lib/perm/auth.py @@ -0,0 +1,102 @@ +# -*- coding:utf-8 -*- + + +from functools import wraps + +import jwt +from flask import current_app +from flask import request +from flask import session +from flask import g +from flask import abort +from flask_login import login_user + +from api.models.account import User +from api.models.account import UserCache + + +def _auth_with_key(): + key = request.values.get('_key') + secret = request.values.get('_secret') + path = request.path + keys = sorted(request.values.keys()) + req_args = [request.values[k] for k in keys if str(k) not in ("_key", "_secret")] + user, authenticated = User.query.authenticate_with_key(key, secret, req_args, path) + if user and authenticated: + login_user(user) + return True + return False + + +def _auth_with_session(): + if isinstance(getattr(g, 'user', None), User): + login_user(g.user) + return True + if "acl" in session and "userName" in (session["acl"] or {}): + login_user(UserCache.get(session["acl"]["userName"])) + return True + return False + + +def _auth_with_token(): + auth_headers = request.headers.get('Access-Token', '').strip() + if not auth_headers: + return False + + try: + token = auth_headers + data = jwt.decode(token, current_app.config['SECRET_KEY']) + user = User.query.filter_by(email=data['sub']).first() + if not user: + return False + + login_user(user) + return True + except jwt.ExpiredSignatureError: + return False + except (jwt.InvalidTokenError, Exception) as e: + return False + + +def _auth_with_ip_white_list(): + ip = request.remote_addr + key = request.values.get('_key') + secret = request.values.get('_secret') + + if not key and not secret and ip.strip() in current_app.config.get("WHITE_LIST", []): # TODO + user = UserCache.get("worker") + login_user(user) + return True + return False + + +def auth_required(func): + if request.json is not None: + setattr(request, 'values', request.json) + else: + setattr(request, 'values', request.values.to_dict()) + + current_app.logger.debug(request.values) + + @wraps(func) + def wrapper(*args, **kwargs): + + if not getattr(func, 'authenticated', True): + return func(*args, **kwargs) + + if _auth_with_session() or _auth_with_key() or _auth_with_token() or _auth_with_ip_white_list(): + return func(*args, **kwargs) + + abort(401) + + return wrapper + + +def auth_abandoned(func): + setattr(func, "authenticated", False) + + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper diff --git a/lib/utils.py b/api/lib/utils.py similarity index 53% rename from lib/utils.py rename to api/lib/utils.py index 944a6d1..dd00fb1 100644 --- a/lib/utils.py +++ b/api/lib/utils.py @@ -1,14 +1,38 @@ -# -*- coding:utf-8 -*- - +# -*- coding:utf-8 -*- +import six import redis - from flask import current_app +def get_page(page): + try: + page = int(page) + except ValueError: + page = 1 + return page if page >= 1 else 1 + + +def get_page_size(page_size): + if page_size == "all": + return page_size + + try: + page_size = int(page_size) + except (ValueError, TypeError): + page_size = current_app.config.get("DEFAULT_PAGE_COUNT") + return page_size if page_size >= 1 else current_app.config.get("DEFAULT_PAGE_COUNT") + + +def handle_arg_list(arg): + return list(filter(lambda x: x != "", arg.strip().split(","))) if isinstance(arg, six.string_types) else arg + + class RedisHandler(object): - def __init__(self, flask_app=None): + def __init__(self, flask_app=None, prefix=None): self.flask_app = flask_app + self.prefix = prefix + self.r = None def init_app(self, app): self.flask_app = app @@ -21,55 +45,30 @@ class RedisHandler(object): db=config.get("REDIS_DB")) self.r = redis.Redis(connection_pool=pool) except Exception as e: + current_app.logger.warning(str(e)) current_app.logger.error("init redis connection failed") - # @classmethod - # def instance(cls): - # if not hasattr(cls, "_instance"): - # cls._instance = cls() - # return cls._instance - - def get(self, ci_ids, key="CMDB_CI"): + def get(self, key_ids): try: - value = self.r.hmget(key, ci_ids) + value = self.r.hmget(self.prefix, key_ids) except Exception as e: current_app.logger.error("get redis error, %s" % str(e)) return return value - def _set(self, ci, key="CMDB_CI"): + def _set(self, obj): try: - self.r.hmset(key, ci) + self.r.hmset(self.prefix, obj) except Exception as e: current_app.logger.error("set redis error, %s" % str(e)) - def add(self, ci): - self._set(ci) + def add(self, obj): + self._set(obj) - def delete(self, ci_id, key="CMDB_CI"): + def delete(self, key_id): try: - ret = self.r.hdel(key, ci_id) + ret = self.r.hdel(self.prefix, key_id) if not ret: - current_app.logger.warn("ci [%d] is not in redis" % ci_id) + current_app.logger.warn("[%d] is not in redis" % key_id) except Exception as e: current_app.logger.error("delete redis key error, %s" % str(e)) - - -def get_page(page): - try: - page = int(page) - except ValueError: - page = 1 - if page < 1: - page = 1 - return page - - -def get_per_page(per_page): - try: - per_page = int(per_page) - except: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - if per_page < 1: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - return per_page \ No newline at end of file diff --git a/api/models/__init__.py b/api/models/__init__.py new file mode 100644 index 0000000..fabbe66 --- /dev/null +++ b/api/models/__init__.py @@ -0,0 +1,5 @@ +# -*- coding:utf-8 -*- + + +from .account import User +from .cmdb import * diff --git a/models/account.py b/api/models/account.py similarity index 69% rename from models/account.py rename to api/models/account.py index 0a3b322..807476e 100644 --- a/models/account.py +++ b/api/models/account.py @@ -1,51 +1,24 @@ # -*- coding:utf-8 -*- - -import hashlib import copy +import hashlib from datetime import datetime -from werkzeug.utils import cached_property -from flask.ext.sqlalchemy import BaseQuery -from flask.ext.principal import RoleNeed -from flask.ext.principal import UserNeed -from flask.ext.principal import Permission +import six +from flask import current_app +from flask_sqlalchemy import BaseQuery -from extensions import db -from extensions import cache -from permissions import admin -from models import row2dict +from api.extensions import db +from api.extensions import cache +from api.lib.database import CRUDModel class UserQuery(BaseQuery): - def from_identity(self, identity): - """ - Loads user from flask.ext.principal.Identity instance and - assigns permissions from user. - - A "user" instance is monkey patched to the identity instance. - - If no user found then None is returned. - """ - - try: - _id = identity.id - if _id: - _id = int(_id) - user = self.get(_id) - except ValueError: - user = None - except Exception: - user = None - if user: - identity.provides.update(user.provides) - identity.user = user - return user - def authenticate(self, login, password): user = self.filter(db.or_(User.username == login, User.email == login)).first() if user: + current_app.logger.info(user) authenticated = user.check_password(password) else: authenticated = False @@ -60,7 +33,7 @@ class UserQuery(BaseQuery): authenticated = True else: authenticated = False - return row2dict(user), authenticated + return user, authenticated def search(self, key): query = self.filter(db.or_(User.email == key, @@ -80,16 +53,14 @@ class UserQuery(BaseQuery): user = self.filter(User.uid == uid).first() return copy.deepcopy(user) - def is_exits(self, username): - user = self.filter(User.username == username).first() - return user is not None - -class User(db.Model): +class User(CRUDModel): __tablename__ = 'users' + __bind_key__ = "user" query_class = UserQuery ADMIN = 1 + OP = 2 uid = db.Column(db.Integer, primary_key=True, autoincrement=True) username = db.Column(db.String(32), unique=True) @@ -98,21 +69,15 @@ class User(db.Model): catalog = db.Column(db.String(64)) email = db.Column(db.String(100), unique=True, nullable=False) mobile = db.Column(db.String(14), unique=True) - _password = db.Column("password", db.String(80), nullable=False) + _password = db.Column("password", db.String(80)) key = db.Column(db.String(32), nullable=False) secret = db.Column(db.String(32), nullable=False) date_joined = db.Column(db.DateTime, default=datetime.utcnow) last_login = db.Column(db.DateTime, default=datetime.utcnow) block = db.Column(db.Boolean, default=False) has_logined = db.Column(db.Boolean, default=False) - - class Permissions(object): - def __init__(self, obj): - self.obj = obj - - @cached_property - def is_admin(self): - return Permission(UserNeed(self.obj.id)) & admin + wx_id = db.Column(db.String(32)) + avatar = db.Column(db.String(128)) def __init__(self, *args, **kwargs): super(User, self).__init__(*args, **kwargs) @@ -120,31 +85,31 @@ class User(db.Model): def __str__(self): return self.username - @cached_property - def permissions(self): - return self.Permissions(self) + def is_active(self): + return not self.block + + def get_id(self): + return self.uid + + @staticmethod + def is_authenticated(): + return True def _get_password(self): return self._password def _set_password(self, password): - self._password = password + self._password = hashlib.md5(password).hexdigest() - password = db.synonym("_password", descriptor=property( - _get_password, _set_password)) + password = db.synonym("_password", + descriptor=property(_get_password, + _set_password)) def check_password(self, password): + if self.password is None: + return False return self.password == password - @cached_property - def provides(self): - needs = [RoleNeed('authenticated'), UserNeed(self.uid)] - for r in self.rolenames: - needs.append(RoleNeed(r)) - if self.is_admin: - needs.append(RoleNeed('admin')) - return needs - @property def roles(self): urs = db.session.query(UserRole.rid).filter( @@ -153,23 +118,28 @@ class User(db.Model): @property def rolenames(self): - return [db.session.query(Role.role_name).filter( - Role.rid == rid).first().role_name for rid in self.roles] + roles = list() + for rid in self.roles: + role = db.session.query(Role).filter(Role.rid == rid).first() + roles.append(role.role_name) + return roles @property def is_admin(self): return self.ADMIN in self.roles -class Role(db.Model): +class Role(CRUDModel): __tablename__ = 'roles' + __bind_key__ = "user" rid = db.Column(db.Integer, primary_key=True, autoincrement=True) role_name = db.Column(db.String(64), nullable=False, unique=True) -class UserRole(db.Model): +class UserRole(CRUDModel): __tablename__ = 'users_roles' + __bind_key__ = "user" uid = db.Column(db.Integer, db.ForeignKey('users.uid'), primary_key=True) rid = db.Column(db.Integer, db.ForeignKey('roles.rid'), primary_key=True) @@ -211,7 +181,7 @@ class RoleCache(object): if not role: role = db.session.query(Role).filter(Role.rid == rid).first() cls.set(role) - elif isinstance(rid, basestring): + elif isinstance(rid, six.string_types): role = cache.get("Role::role_name::%s" % rid) if not role: role = db.session.query(Role).filter( @@ -227,4 +197,4 @@ class RoleCache(object): @classmethod def clean(cls, role): cache.delete("Role::rid::%s" % role.rid, role) - cache.delete("Role::role_name::%s" % role.role_name, role) \ No newline at end of file + cache.delete("Role::role_name::%s" % role.role_name, role) diff --git a/api/models/cmdb.py b/api/models/cmdb.py new file mode 100644 index 0000000..37251c5 --- /dev/null +++ b/api/models/cmdb.py @@ -0,0 +1,325 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from api.lib.database import Model +from api.extensions import db + + +# template + +class RelationType(Model): + __tablename__ = "c_relation_types" + + name = db.Column(db.String(16), index=True) + + +class CITypeGroup(Model): + __tablename__ = "c_ci_type_groups" + + name = db.Column(db.String(32)) + + +class CITypeGroupItem(Model): + __tablename__ = "c_ci_type_group_items" + + group_id = db.Column(db.Integer, db.ForeignKey("c_ci_type_groups.id"), nullable=False) + type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + order = db.Column(db.SmallInteger, default=0) + + +class CIType(Model): + __tablename__ = "c_ci_types" + + name = db.Column(db.String(32)) + alias = db.Column(db.String(32)) + unique_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False) + enabled = db.Column(db.Boolean, default=True, nullable=False) + is_attached = db.Column(db.Boolean, default=False, nullable=False) + icon_url = db.Column(db.String(256)) + order = db.Column(db.SmallInteger, default=0, nullable=False) + + unique_key = db.relationship("Attribute", backref="c_ci_types.unique_id") + + +class CITypeRelation(Model): + __tablename__ = "c_ci_type_relations" + + parent_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + child_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False) + + parent = db.relationship("CIType", primaryjoin="CIType.id==CITypeRelation.parent_id") + child = db.relationship("CIType", primaryjoin="CIType.id==CITypeRelation.child_id") + relation_type = db.relationship("RelationType", backref="c_ci_type_relations.relation_type_id") + + +class Attribute(Model): + __tablename__ = "c_attributes" + + INT = "0" + FLOAT = "1" + TEXT = "2" + DATETIME = "3" + DATE = "4" + TIME = "5" + + name = db.Column(db.String(32), nullable=False) + alias = db.Column(db.String(32), nullable=False) + value_type = db.Column(db.Enum(INT, FLOAT, TEXT, DATETIME, DATE, TIME), default=TEXT, nullable=False) + + is_choice = db.Column(db.Boolean, default=False) + is_list = db.Column(db.Boolean, default=False) + is_unique = db.Column(db.Boolean, default=False) + is_index = db.Column(db.Boolean, default=False) + is_link = db.Column(db.Boolean, default=False) + is_password = db.Column(db.Boolean, default=False) + is_sortable = db.Column(db.Boolean, default=False) + + +class CITypeAttribute(Model): + __tablename__ = "c_ci_type_attributes" + + type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False) + order = db.Column(db.Integer, default=0) + is_required = db.Column(db.Boolean, default=False) + default_show = db.Column(db.Boolean, default=True) + + attr = db.relationship("Attribute", backref="c_ci_type_attributes.attr_id") + + +class CITypeAttributeGroup(Model): + __tablename__ = "c_ci_type_attribute_groups" + + name = db.Column(db.String(64)) + type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + order = db.Column(db.SmallInteger, default=0) + + +class CITypeAttributeGroupItem(Model): + __tablename__ = "c_ci_type_attribute_group_items" + + group_id = db.Column(db.Integer, db.ForeignKey("c_ci_type_attribute_groups.id"), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False) + order = db.Column(db.SmallInteger, default=0) + + +# instance + +class CI(Model): + __tablename__ = "c_cis" + + REVIEW = "0" + VALIDATE = "1" + + type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + status = db.Column(db.Enum(REVIEW, VALIDATE, name="status")) + heartbeat = db.Column(db.DateTime, default=lambda: datetime.datetime.now()) + + ci_type = db.relationship("CIType", backref="c_cis.type_id") + + +class CIRelation(Model): + __tablename__ = "c_ci_relations" + + first_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False) + second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False) + relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False) + more = db.Column(db.Integer, db.ForeignKey("c_cis.id")) + + first_ci = db.relationship("CI", primaryjoin="CI.id==CIRelation.first_ci_id") + second_ci = db.relationship("CI", primaryjoin="CI.id==CIRelation.second_ci_id") + relation_type = db.relationship("RelationType", backref="c_ci_relations.relation_type_id") + + +class IntegerChoice(Model): + __tablename__ = 'c_choice_integers' + + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Integer, nullable=False) + + attr = db.relationship("Attribute", backref="c_choice_integers.attr_id") + + +class FloatChoice(Model): + __tablename__ = 'c_choice_floats' + + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Float, nullable=False) + + attr = db.relationship("Attribute", backref="c_choice_floats.attr_id") + + +class TextChoice(Model): + __tablename__ = 'c_choice_texts' + + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Text, nullable=False) + + attr = db.relationship("Attribute", backref="c_choice_texts.attr_id") + + +class CIIndexValueInteger(Model): + __tablename__ = "c_value_index_integers" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Integer, nullable=False) + + ci = db.relationship("CI", backref="c_value_index_integers.ci_id") + attr = db.relationship("Attribute", backref="c_value_index_integers.attr_id") + + __table_args__ = (db.Index("integer_attr_value_index", "attr_id", "value"), ) + + +class CIIndexValueFloat(Model): + __tablename__ = "c_value_index_floats" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Float, nullable=False) + + ci = db.relationship("CI", backref="c_value_index_floats.ci_id") + attr = db.relationship("Attribute", backref="c_value_index_floats.attr_id") + + __table_args__ = (db.Index("float_attr_value_index", "attr_id", "value"), ) + + +class CIIndexValueText(Model): + __tablename__ = "c_value_index_texts" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.String(128), nullable=False) + + ci = db.relationship("CI", backref="c_value_index_texts.ci_id") + attr = db.relationship("Attribute", backref="c_value_index_texts.attr_id") + + __table_args__ = (db.Index("text_attr_value_index", "attr_id", "value"), ) + + +class CIIndexValueDateTime(Model): + __tablename__ = "c_value_index_datetime" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.DateTime, nullable=False) + + ci = db.relationship("CI", backref="c_value_index_datetime.ci_id") + attr = db.relationship("Attribute", backref="c_value_index_datetime.attr_id") + + __table_args__ = (db.Index("datetime_attr_value_index", "attr_id", "value"), ) + + +class CIValueInteger(Model): + __tablename__ = "c_value_integers" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Integer, nullable=False) + + ci = db.relationship("CI", backref="c_value_integers.ci_id") + attr = db.relationship("Attribute", backref="c_value_integers.attr_id") + + +class CIValueFloat(Model): + __tablename__ = "c_value_floats" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Float, nullable=False) + + ci = db.relationship("CI", backref="c_value_floats.ci_id") + attr = db.relationship("Attribute", backref="c_value_floats.attr_id") + + +class CIValueText(Model): + __tablename__ = "c_value_texts" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Text, nullable=False) + + ci = db.relationship("CI", backref="c_value_texts.ci_id") + attr = db.relationship("Attribute", backref="c_value_texts.attr_id") + + +class CIValueDateTime(Model): + __tablename__ = "c_value_datetime" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.DateTime, nullable=False) + + ci = db.relationship("CI", backref="c_value_datetime.ci_id") + attr = db.relationship("Attribute", backref="c_value_datetime.attr_id") + + +# history +class OperationRecord(Model): + __tablename__ = "c_records" + + uid = db.Column(db.Integer, index=True, nullable=False) + origin = db.Column(db.String(32)) + ticket_id = db.Column(db.String(32)) + reason = db.Column(db.Text) + + +class AttributeHistory(Model): + __tablename__ = "c_attribute_histories" + + ADD = "0" + DELETE = "1" + UPDATE = "2" + + operate_type = db.Column(db.Enum(ADD, DELETE, UPDATE, name="operate_type")) + record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False) + ci_id = db.Column(db.Integer, index=True, nullable=False) + attr_id = db.Column(db.Integer, index=True) + old = db.Column(db.Text) + new = db.Column(db.Text) + + +class CIRelationHistory(Model): + __tablename__ = "c_relation_histories" + + ADD = "0" + DELETE = "1" + + operate_type = db.Column(db.Enum(ADD, DELETE, name="operate_type")) + record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False) + first_ci_id = db.Column(db.Integer) + second_ci_id = db.Column(db.Integer) + relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id")) + relation_id = db.Column(db.Integer, nullable=False) + + +# preference +class PreferenceShowAttributes(Model): + __tablename__ = "c_preference_show_attributes" + + uid = db.Column(db.Integer, index=True, nullable=False) + type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id")) + order = db.Column(db.SmallInteger, default=0) + + ci_type = db.relationship("CIType", backref="c_preference_show_attributes.type_id") + attr = db.relationship("Attribute", backref="c_preference_show_attributes.attr_id") + + +class PreferenceTreeView(Model): + __tablename__ = "c_preference_tree_views" + + uid = db.Column(db.Integer, index=True, nullable=False) + type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + levels = db.Column(db.Text) # TODO: JSON + + +class PreferenceRelationView(Model): + __tablename__ = "c_preference_relation_views" + + name = db.Column(db.String(8), index=True, nullable=False) + parent_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) + child_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) diff --git a/api/resource.py b/api/resource.py new file mode 100644 index 0000000..a496e0b --- /dev/null +++ b/api/resource.py @@ -0,0 +1,45 @@ +# -*- coding:utf-8 -*- + +import os +import sys +from inspect import getmembers, isclass + +import six +from flask import jsonify +from flask_restful import Resource + +from api.lib.perm.auth import auth_required + + +class APIView(Resource): + method_decorators = [auth_required] + + def __init__(self): + super(APIView, self).__init__() + + @staticmethod + def jsonify(*args, **kwargs): + return jsonify(*args, **kwargs) + + +API_PACKAGE = "api" + + +def register_resources(resource_path, rest_api): + for root, _, files in os.walk(os.path.join(resource_path)): + for filename in files: + if not filename.startswith("_") and filename.endswith("py"): + module_path = os.path.join(API_PACKAGE, root[root.index("views"):]) + if module_path not in sys.path: + sys.path.insert(1, module_path) + view = __import__(os.path.splitext(filename)[0]) + resource_list = [o[0] for o in getmembers(view) if isclass(o[1]) and issubclass(o[1], Resource)] + resource_list = [i for i in resource_list if i != "APIView"] + for resource_cls_name in resource_list: + resource_cls = getattr(view, resource_cls_name) + if not hasattr(resource_cls, "url_prefix"): + resource_cls.url_prefix = ("",) + if isinstance(resource_cls.url_prefix, six.string_types): + resource_cls.url_prefix = (resource_cls.url_prefix,) + + rest_api.add_resource(resource_cls, *resource_cls.url_prefix) diff --git a/api/settings.py.example b/api/settings.py.example new file mode 100644 index 0000000..c9315fe --- /dev/null +++ b/api/settings.py.example @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +"""Application configuration. + +Most configuration is set via environment variables. + +For local development, use a .env file to set +environment variables. +""" +from environs import Env + +env = Env() +env.read_env() + +ENV = env.str("FLASK_ENV", default="production") +DEBUG = ENV == "development" +SECRET_KEY = env.str("SECRET_KEY") +BCRYPT_LOG_ROUNDS = env.int("BCRYPT_LOG_ROUNDS", default=13) +DEBUG_TB_ENABLED = DEBUG +DEBUG_TB_INTERCEPT_REDIRECTS = False + + +ERROR_CODES = [400, 401, 403, 404, 405, 500, 502] + +# # database +SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{db}?charset=utf8' +SQLALCHEMY_BINDS = { + "user": 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{db}?charset=utf8' +} +SQLALCHEMY_ECHO = False +SQLALCHEMY_TRACK_MODIFICATIONS = False +SQLALCHEMY_ENGINE_OPTIONS = { + 'pool_recycle': 300, +} + +# # cache +CACHE_TYPE = "redis" +CACHE_REDIS_HOST = "127.0.0.1" +CACHE_REDIS_PORT = 6379 +CACHE_KEY_PREFIX = "CMDB::" +CACHE_DEFAULT_TIMEOUT = 3000 + +# # log +LOG_PATH = './logs/app.log' +LOG_LEVEL = 'DEBUG' + + +# # mail +MAIL_SERVER = '' +MAIL_PORT = 25 +MAIL_USE_TLS = False +MAIL_USE_SSL = False +MAIL_DEBUG = True +MAIL_USERNAME = '' +MAIL_PASSWORD = '' +DEFAULT_MAIL_SENDER = '' + +# # queue +CELERY_RESULT_BACKEND = "redis://127.0.0.1:6379/2" +BROKER_URL = 'redis://127.0.0.1:6379/2' +BROKER_VHOST = '/' + + +# # SSO +CAS_SERVER = "http://sso.xxx.com" +CAS_VALIDATE_SERVER = "http://sso.xxx.com" +CAS_LOGIN_ROUTE = "/cas/login" +CAS_LOGOUT_ROUTE = "/cas/logout" +CAS_VALIDATE_ROUTE = "/cas/serviceValidate" +CAS_AFTER_LOGIN = "/" +DEFAULT_SERVICE = "http://127.0.0.1:8000" + +# # pagination +DEFAULT_PAGE_COUNT = 50 + +# # permission +WHITE_LIST = ["127.0.0.1"] +USE_ACL = False diff --git a/api/tasks/__init__.py b/api/tasks/__init__.py new file mode 100644 index 0000000..3b26f17 --- /dev/null +++ b/api/tasks/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/tasks/cmdb.py b/api/tasks/cmdb.py similarity index 51% rename from tasks/cmdb.py rename to api/tasks/cmdb.py index 45a13e1..d48943a 100644 --- a/tasks/cmdb.py +++ b/api/tasks/cmdb.py @@ -6,24 +6,27 @@ import time from flask import current_app -from extensions import celery -from extensions import db -from extensions import rd -import lib.ci +import api.lib.cmdb.ci +from api.extensions import celery +from api.extensions import db +from api.extensions import rd +from api.lib.cmdb.const import CMDB_QUEUE -@celery.task(name="cmdb.ci_cache", queue="cmdb_async") +@celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE) def ci_cache(ci_id): time.sleep(0.1) db.session.close() - m = lib.ci.CIManager() - ci = m.get_ci_by_id(ci_id, need_children=False, use_master=True) + + m = api.lib.cmdb.ci.CIManager() + ci = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) rd.delete(ci_id) rd.add({ci_id: json.dumps(ci)}) + current_app.logger.info("%d caching.........." % ci_id) -@celery.task(name="cmdb.ci_delete", queue="cmdb_async") +@celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE) def ci_delete(ci_id): current_app.logger.info(ci_id) rd.delete(ci_id) diff --git a/api/tasks/test.py b/api/tasks/test.py new file mode 100644 index 0000000..a19d362 --- /dev/null +++ b/api/tasks/test.py @@ -0,0 +1,9 @@ +# -*- coding:utf-8 -*- + +from api.extensions import celery +from flask import current_app + + +@celery.task(queue="ticket_web") +def test_task(): + current_app.logger.info("test task.............................") diff --git a/api/views/__init__.py b/api/views/__init__.py new file mode 100644 index 0000000..3db020b --- /dev/null +++ b/api/views/__init__.py @@ -0,0 +1,31 @@ +# -*- coding:utf-8 -*- + +import os + +from flask import Blueprint +from flask_restful import Api + +from api.resource import register_resources +from .permission import GetResourcesView, HasPermissionView, GetUserInfoView +from .account import LoginView, LogoutView + +HERE = os.path.abspath(os.path.dirname(__file__)) + +# account +blueprint_account = Blueprint('account_api', __name__, url_prefix='/api') +account_rest = Api(blueprint_account) +account_rest.add_resource(LoginView, LoginView.url_prefix) +account_rest.add_resource(LogoutView, LogoutView.url_prefix) + +# permission +blueprint_perm_v01 = Blueprint('permission_api', __name__, url_prefix='/api/v1/perms') +perm_rest = Api(blueprint_perm_v01) +perm_rest.add_resource(GetResourcesView, GetResourcesView.url_prefix) +perm_rest.add_resource(HasPermissionView, HasPermissionView.url_prefix) +perm_rest.add_resource(GetUserInfoView, GetUserInfoView.url_prefix) + + +# cmdb +blueprint_cmdb_v01 = Blueprint('cmdb_api_v01', __name__, url_prefix='/api/v0.1') +rest = Api(blueprint_cmdb_v01) +register_resources(os.path.join(HERE, "cmdb"), rest) diff --git a/api/views/account.py b/api/views/account.py new file mode 100644 index 0000000..4b1d9dc --- /dev/null +++ b/api/views/account.py @@ -0,0 +1,47 @@ +# -*- coding:utf-8 -*- + +import datetime + +import jwt +from flask import request +from flask import current_app +from flask import abort +from flask_login import login_user, logout_user + +from api.resource import APIView +from api.lib.decorator import args_required +from api.lib.perm.auth import auth_abandoned +from api.models.account import User + + +class LoginView(APIView): + url_prefix = "/login" + + @args_required("username") + @args_required("password") + @auth_abandoned + def post(self): + username = request.values.get("username") or request.values.get("email") + password = request.values.get("password") + user, authenticated = User.query.authenticate(username, password) + if not authenticated: + return abort(401, "invalid username or password") + + login_user(user) + + token = jwt.encode({ + 'sub': user.email, + 'iat': datetime.datetime.now(), + 'exp': datetime.datetime.now() + datetime.timedelta(minutes=24 * 60 * 7)}, + current_app.config['SECRET_KEY']) + + return self.jsonify(token=token.decode()) + + +class LogoutView(APIView): + url_prefix = "/logout" + + @auth_abandoned + def post(self): + logout_user() + self.jsonify(code=200) diff --git a/api/views/cmdb/__init__.py b/api/views/cmdb/__init__.py new file mode 100644 index 0000000..3b26f17 --- /dev/null +++ b/api/views/cmdb/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/api/views/cmdb/attribute.py b/api/views/cmdb/attribute.py new file mode 100644 index 0000000..0ce947b --- /dev/null +++ b/api/views/cmdb/attribute.py @@ -0,0 +1,65 @@ +# -*- coding:utf-8 -*- + + +from flask import request +from flask import abort +from flask import current_app + +from api.resource import APIView +from api.lib.perm.acl import role_required +from api.lib.cmdb.const import RoleEnum +from api.lib.cmdb.attribute import AttributeManager +from api.lib.decorator import args_required +from api.lib.utils import handle_arg_list + + +class AttributeSearchView(APIView): + url_prefix = ("/attributes/s", "/attributes/search") + + def get(self): + q = request.values.get("q") + attrs = AttributeManager().get_attributes(name=q) + count = len(attrs) + return self.jsonify(numfound=count, attributes=attrs) + + +class AttributeView(APIView): + url_prefix = ("/attributes", "/attributes/", "/attributes/") + + def get(self, attr_name=None, attr_id=None): + attr_manager = AttributeManager() + attr_dict = None + if attr_name is not None: + attr_dict = attr_manager.get_attribute_by_name(attr_name) + if attr_dict is None: + attr_dict = attr_manager.get_attribute_by_alias(attr_name) + elif attr_id is not None: + attr_dict = attr_manager.get_attribute_by_id(attr_id) + if attr_dict is not None: + return self.jsonify(attribute=attr_dict) + abort(404, "Attribute is not found") + + @role_required(RoleEnum.CONFIG) + @args_required("name") + def post(self): + choice_value = handle_arg_list(request.values.get("choice_value")) + params = request.values + params["choice_value"] = choice_value + current_app.logger.debug(params) + + attr_id = AttributeManager.add(**params) + return self.jsonify(attr_id=attr_id) + + @role_required(RoleEnum.CONFIG) + def put(self, attr_id): + choice_value = handle_arg_list(request.values.get("choice_value")) + params = request.values + params["choice_value"] = choice_value + current_app.logger.debug(params) + AttributeManager().update(attr_id, **params) + return self.jsonify(attr_id=attr_id) + + @role_required(RoleEnum.CONFIG) + def delete(self, attr_id): + attr_name = AttributeManager.delete(attr_id) + return self. jsonify(message="attribute {0} deleted".format(attr_name)) diff --git a/api/views/cmdb/ci.py b/api/views/cmdb/ci.py new file mode 100644 index 0000000..0f6bd16 --- /dev/null +++ b/api/views/cmdb/ci.py @@ -0,0 +1,218 @@ +# -*- coding:utf-8 -*- + +import time + +import six +from flask import abort +from flask import current_app +from flask import request + +from api.lib.perm.acl import has_perm_from_args +from api.lib.cmdb.const import ResourceType, PermEnum +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.ci import CIManager +from api.lib.cmdb.const import ExistPolicy +from api.lib.cmdb.const import RetKey +from api.lib.cmdb.search import Search +from api.lib.cmdb.search import SearchError +from api.lib.perm.auth import auth_abandoned +from api.lib.utils import get_page +from api.lib.utils import get_page_size +from api.lib.utils import handle_arg_list +from api.models.cmdb import CI +from api.resource import APIView + + +class CIsByTypeView(APIView): + url_prefix = "/ci/type/" + + def get(self, type_id): + fields = handle_arg_list(request.values.get("fields", "")) + + ret_key = request.values.get("ret_key", RetKey.NAME) + if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID): + ret_key = RetKey.NAME + + page = get_page(request.values.get("page", 1)) + count = get_page_size(request.values.get("count")) + + manager = CIManager() + res = manager.get_cis_by_type(type_id, + ret_key=ret_key, + fields=fields, + page=page, + per_page=count) + + return self.jsonify(type_id=type_id, + numfound=res[0], + total=len(res[2]), + page=res[1], + cis=res[2]) + + +class CIView(APIView): + url_prefix = ("/ci/", "/ci") + + def get(self, ci_id): + fields = handle_arg_list(request.values.get("fields", "")) + + ret_key = request.values.get("ret_key", RetKey.NAME) + if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID): + ret_key = RetKey.NAME + + manager = CIManager() + ci = manager.get_ci_by_id_from_db(ci_id, ret_key=ret_key, fields=fields) + return self.jsonify(ci_id=ci_id, ci=ci) + + @staticmethod + def _wrap_ci_dict(): + ci_dict = dict() + for k, v in request.values.items(): + if k != "ci_type" and not k.startswith("_"): + ci_dict[k] = v.strip() if isinstance(v, six.string_types) else v + return ci_dict + + @has_perm_from_args("ci_type", ResourceType.CI, PermEnum.ADD) + def post(self): + ci_type = request.values.get("ci_type") + _no_attribute_policy = request.values.get("_no_attribute_policy", ExistPolicy.IGNORE) + + ci_dict = self._wrap_ci_dict() + + manager = CIManager() + current_app.logger.debug(ci_dict) + ci_id = manager.add(ci_type, + exist_policy=ExistPolicy.REJECT, + _no_attribute_policy=_no_attribute_policy, **ci_dict) + return self.jsonify(ci_id=ci_id) + + @has_perm_from_args("ci_id", ResourceType.CI, PermEnum.UPDATE, CIManager.get_type_name) + def put(self, ci_id=None): + args = request.values + ci_type = args.get("ci_type") + _no_attribute_policy = args.get("_no_attribute_policy", ExistPolicy.IGNORE) + + ci_dict = self._wrap_ci_dict() + manager = CIManager() + if ci_id is not None: + manager.update(ci_id, **ci_dict) + else: + ci_id = manager.add(ci_type, + exist_policy=ExistPolicy.REPLACE, + _no_attribute_policy=_no_attribute_policy, + **ci_dict) + return self.jsonify(ci_id=ci_id) + + @has_perm_from_args("ci_id", ResourceType.CI, PermEnum.DELETE, CIManager.get_type_name) + def delete(self, ci_id): + manager = CIManager() + manager.delete(ci_id) + return self.jsonify(message="ok") + + +class CIDetailView(APIView): + url_prefix = "/ci//detail" + + def get(self, ci_id): + _ci = CI.get_by_id(ci_id).to_dict() + return self.jsonify(**_ci) + + +class CISearchView(APIView): + url_prefix = ("/ci/s", "/ci/search") + + @auth_abandoned + def get(self): + """@params: q: query statement + fl: filter by column + count: the number of ci + ret_key: id, name, alias + facet: statistic + """ + + page = get_page(request.values.get("page", 1)) + count = get_page_size(request.values.get("count")) + + query = request.values.get('q', "") + fl = handle_arg_list(request.values.get('fl', "")) + ret_key = request.values.get('ret_key', RetKey.NAME) + if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID): + ret_key = RetKey.NAME + facet = handle_arg_list(request.values.get("facet", "")) + fl = list(filter(lambda x: x != "", fl)) + facet = list(filter(lambda x: x != "", facet)) + sort = request.values.get("sort") + + start = time.time() + s = Search(query, fl, facet, page, ret_key, count, sort) + try: + response, counter, total, page, numfound, facet = s.search() + except SearchError as e: + return abort(400, str(e)) + except Exception as e: + current_app.logger.error(str(e)) + return abort(500, "search unknown error") + current_app.logger.debug("search time is :{0}".format(time.time() - start)) + return self.jsonify(numfound=numfound, + total=total, + page=page, + facet=facet, + counter=counter, + result=response) + + +class CIUnique(APIView): + url_prefix = "/ci//unique" + + @has_perm_from_args("ci_id", ResourceType.CI, PermEnum.UPDATE, CIManager.get_type_name) + def put(self, ci_id): + params = request.values + unique_name = params.keys()[0] + unique_value = params.values()[0] + + CIManager.update_unique_value(ci_id, unique_name, unique_value) + + return self.jsonify(ci_id=ci_id) + + +class CIHeartbeatView(APIView): + url_prefix = ("/ci/heartbeat", "/ci/heartbeat//") + + def get(self): + page = get_page(request.values.get("page", 1)) + ci_type = request.values.get("ci_type", "").strip() + try: + type_id = CITypeCache.get(ci_type).type_id + except AttributeError: + return self.jsonify(numfound=0, result=[]) + agent_status = request.values.get("agent_status") + if agent_status: + agent_status = int(agent_status) + + numfound, result = CIManager.get_heartbeat(page, type_id, agent_status=agent_status) + + return self.jsonify(numfound=numfound, result=result) + + def post(self, ci_type, unique): + if not unique or not ci_type: + return self.jsonify(message="error") + + msg, cmd = CIManager().add_heartbeat(ci_type, unique) + + return self.jsonify(message=msg, cmd=cmd) + + +class CIFlushView(APIView): + url_prefix = ("/ci/flush", "/ci//flush") + + @auth_abandoned + def get(self, ci_id=None): + from api.tasks.cmdb import ci_cache + from api.lib.cmdb.const import CMDB_QUEUE + if ci_id is not None: + ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + else: + cis = CI.get_by(to_dict=False) + for ci in cis: + ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) + return self.jsonify(code=200) diff --git a/api/views/cmdb/ci_relation.py b/api/views/cmdb/ci_relation.py new file mode 100644 index 0000000..86639e3 --- /dev/null +++ b/api/views/cmdb/ci_relation.py @@ -0,0 +1,66 @@ +# -*- coding:utf-8 -*- + + +from flask import request + +from api.lib.cmdb.ci import CIRelationManager +from api.lib.utils import get_page +from api.lib.utils import get_page_size +from api.resource import APIView + + +class GetSecondCIsView(APIView): + url_prefix = "/ci_relations//second_cis" + + def get(self, first_ci_id): + page = get_page(request.values.get("page", 1)) + count = get_page_size(request.values.get("count")) + relation_type = request.values.get("relation_type", "contain") + + manager = CIRelationManager() + numfound, total, second_cis = manager.get_second_cis( + first_ci_id, page=page, per_page=count, relation_type=relation_type) + + return self.jsonify(numfound=numfound, + total=total, + page=page, + second_cis=second_cis) + + +class GetFirstCIsView(APIView): + url_prefix = "/ci_relations//first_cis" + + def get(self, second_ci_id): + page = get_page(request.values.get("page", 1)) + count = get_page_size(request.values.get("count")) + + manager = CIRelationManager() + numfound, total, first_cis = manager.get_first_cis(second_ci_id, per_page=count, page=page) + + return self.jsonify(numfound=numfound, + total=total, + page=page, + first_cis=first_cis) + + +class CIRelationView(APIView): + url_prefix = "/ci_relations//" + + def post(self, first_ci_id, second_ci_id): + manager = CIRelationManager() + res = manager.add(first_ci_id, second_ci_id) + return self.jsonify(cr_id=res) + + def delete(self, first_ci_id, second_ci_id): + manager = CIRelationManager() + manager.delete_2(first_ci_id, second_ci_id) + return self.jsonify(message="CIType Relation is deleted") + + +class DeleteCIRelationView(APIView): + url_prefix = "/ci_relations/" + + def delete(self, cr_id): + manager = CIRelationManager() + manager.delete(cr_id) + return self.jsonify(message="CIType Relation is deleted") diff --git a/api/views/cmdb/ci_type.py b/api/views/cmdb/ci_type.py new file mode 100644 index 0000000..14b6153 --- /dev/null +++ b/api/views/cmdb/ci_type.py @@ -0,0 +1,202 @@ +# -*- coding:utf-8 -*- + + +from flask import abort +from flask import current_app +from flask import request + +from api.resource import APIView +from api.lib.perm.acl import role_required +from api.lib.cmdb.const import RoleEnum +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.ci_type import CITypeAttributeManager +from api.lib.cmdb.ci_type import CITypeManager +from api.lib.cmdb.ci_type import CITypeGroupManager +from api.lib.cmdb.ci_type import CITypeAttributeGroupManager +from api.lib.decorator import args_required +from api.lib.utils import handle_arg_list + + +class CITypeView(APIView): + url_prefix = ("/ci_types", "/ci_types/", "/ci_types/") + + def get(self, type_id=None, type_name=None): + q = request.args.get("type_name") + + if type_id is not None: + ci_types = [CITypeCache.get(type_id).to_dict()] + elif type_name is not None: + ci_types = [CITypeCache.get(type_name).to_dict()] + else: + ci_types = CITypeManager().get_ci_types(q) + count = len(ci_types) + + return self.jsonify(numfound=count, ci_types=ci_types) + + @role_required(RoleEnum.CONFIG) + @args_required("name") + def post(self): + params = request.values + + type_name = params.get("name") + type_alias = params.get("alias") + type_alias = type_name if not type_alias else type_alias + params['alias'] = type_alias + + manager = CITypeManager() + type_id = manager.add(**params) + + return self.jsonify(type_id=type_id) + + @role_required(RoleEnum.CONFIG) + def put(self, type_id): + params = request.values + + manager = CITypeManager() + manager.update(type_id, **params) + return self.jsonify(type_id=type_id) + + @role_required(RoleEnum.CONFIG) + def delete(self, type_id): + CITypeManager.delete(type_id) + return self.jsonify(type_id=type_id) + + +class CITypeGroupView(APIView): + url_prefix = ("/ci_types/groups", "/ci_types/groups/") + + def get(self): + need_other = request.values.get("need_other") + return self.jsonify(CITypeGroupManager.get(need_other)) + + @role_required(RoleEnum.CONFIG) + @args_required("name") + def post(self): + name = request.values.get("name") + group = CITypeGroupManager.add(name) + return self.jsonify(group.to_dict()) + + @role_required(RoleEnum.CONFIG) + def put(self, gid): + name = request.values.get('name') + type_ids = request.values.get('type_ids') + CITypeGroupManager.update(gid, name, type_ids) + return self.jsonify(gid=gid) + + @role_required(RoleEnum.CONFIG) + def delete(self, gid): + CITypeGroupManager.delete(gid) + return self.jsonify(gid=gid) + + +class CITypeQueryView(APIView): + url_prefix = "/ci_types/query" + + @args_required("q") + def get(self): + q = request.args.get("q") + res = CITypeManager.query(q) + return self.jsonify(ci_type=res) + + +class EnableCITypeView(APIView): + url_prefix = "/ci_types//enable" + + @role_required(RoleEnum.CONFIG) + def post(self, type_id): + enable = request.values.get("enable", True) + CITypeManager.set_enabled(type_id, enabled=enable) + return self.jsonify(type_id=type_id, enable=enable) + + +class CITypeAttributeView(APIView): + url_prefix = ("/ci_types//attributes", "/ci_types//attributes") + + def get(self, type_id=None, type_name=None): + t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, "CIType does not exist") + type_id = t.id + unique_id = t.unique_id + unique = AttributeCache.get(unique_id).name + return self.jsonify(attributes=CITypeAttributeManager.get_attributes_by_type_id(type_id), + type_id=type_id, + unique_id=unique_id, + unique=unique) + + @role_required(RoleEnum.CONFIG) + @args_required("attr_id") + def post(self, type_id=None): + attr_id_list = handle_arg_list(request.values.get("attr_id")) + params = request.values + params.pop("attr_id", "") + + CITypeAttributeManager.add(type_id, attr_id_list, **params) + return self.jsonify(attributes=attr_id_list) + + @role_required(RoleEnum.CONFIG) + @args_required("attributes") + def put(self, type_id=None): + """ + attributes is list, only support raw data request + :param type_id: + :return: + """ + attributes = request.values.get("attributes") + current_app.logger.debug(attributes) + if not isinstance(attributes, list): + return abort(400, "attributes must be list") + CITypeAttributeManager.update(type_id, attributes) + return self.jsonify(attributes=attributes) + + @role_required(RoleEnum.CONFIG) + @args_required("attr_id") + def delete(self, type_id=None): + """ + Form request: attr_id is a string, separated by commas + Raw data request: attr_id is a list + :param type_id: + :return: + """ + attr_id_list = handle_arg_list(request.values.get("attr_id", "")) + + CITypeAttributeManager.delete(type_id, attr_id_list) + + return self.jsonify(attributes=attr_id_list) + + +class CITypeAttributeGroupView(APIView): + url_prefix = ("/ci_types//attribute_groups", + "/ci_types/attribute_groups/") + + def get(self, type_id): + need_other = request.values.get("need_other") + return self.jsonify(CITypeAttributeGroupManager.get_by_type_id(type_id, need_other)) + + @role_required(RoleEnum.CONFIG) + @args_required("name") + def post(self, type_id): + name = request.values.get("name").strip() + order = request.values.get("order") or 0 + attrs = handle_arg_list(request.values.get("attributes", "")) + orders = list(range(len(attrs))) + + attr_order = list(zip(attrs, orders)) + group = CITypeAttributeGroupManager.create_or_update(type_id, name, attr_order, order) + current_app.logger.warning(group.id) + return self.jsonify(group_id=group.id) + + @role_required(RoleEnum.CONFIG) + def put(self, group_id): + name = request.values.get("name") + order = request.values.get("order") or 0 + attrs = handle_arg_list(request.values.get("attributes", "")) + orders = list(range(len(attrs))) + + attr_order = list(zip(attrs, orders)) + CITypeAttributeGroupManager.update(group_id, name, attr_order, order) + return self.jsonify(group_id=group_id) + + @role_required(RoleEnum.CONFIG) + def delete(self, group_id): + CITypeAttributeGroupManager.delete(group_id) + return self.jsonify(group_id=group_id) diff --git a/api/views/cmdb/ci_type_relation.py b/api/views/cmdb/ci_type_relation.py new file mode 100644 index 0000000..613d1ba --- /dev/null +++ b/api/views/cmdb/ci_type_relation.py @@ -0,0 +1,49 @@ +# -*- coding:utf-8 -*- + + +from flask import request + +from api.lib.perm.acl import role_required +from api.lib.cmdb.const import RoleEnum +from api.lib.cmdb.ci_type import CITypeRelationManager +from api.lib.decorator import args_required +from api.resource import APIView + + +class GetChildrenView(APIView): + url_prefix = "/ci_type_relations//children" + + def get(self, parent_id): + return self.jsonify(children=CITypeRelationManager.get_children(parent_id)) + + +class GetParentsView(APIView): + url_prefix = "/ci_type_relations//parents" + + def get(self, child_id): + return self.jsonify(parents=CITypeRelationManager.get_parents(child_id)) + + +class CITypeRelationView(APIView): + url_prefix = "/ci_type_relations//" + + @role_required(RoleEnum.CONFIG) + @args_required("relation_type_id") + def post(self, parent_id, child_id): + relation_type_id = request.values.get("relation_type_id") + ctr_id = CITypeRelationManager.add(parent_id, child_id, relation_type_id) + return self.jsonify(ctr_id=ctr_id) + + @role_required(RoleEnum.CONFIG) + def delete(self, parent_id, child_id): + CITypeRelationManager.delete_2(parent_id, child_id) + return self.jsonify(code=200, parent_id=parent_id, child_id=child_id) + + +class CITypeRelationDelete2View(APIView): + url_prefix = "/ci_type_relations/" + + @role_required(RoleEnum.CONFIG) + def delete(self, ctr_id): + CITypeRelationManager.delete(ctr_id) + return self.jsonify(code=200, ctr_id=ctr_id) diff --git a/api/views/cmdb/history.py b/api/views/cmdb/history.py new file mode 100644 index 0000000..e0029a5 --- /dev/null +++ b/api/views/cmdb/history.py @@ -0,0 +1,63 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from flask import abort +from flask import request + +from api.lib.cmdb.history import AttributeHistoryManger +from api.lib.utils import get_page +from api.lib.utils import get_page_size +from api.resource import APIView + + +class RecordView(APIView): + url_prefix = "/history/records" + + def get(self): + page = get_page(request.values.get("page", 1)) + page_size = get_page_size(request.values.get("page_size")) + _start = request.values.get("start") + _end = request.values.get("end") + username = request.values.get("username", "") + start, end = None, None + if _start: + try: + start = datetime.datetime.strptime(_start, '%Y-%m-%d %H:%M:%S') + except ValueError: + abort(400, 'incorrect start date time') + if _end: + try: + end = datetime.datetime.strptime(_end, '%Y-%m-%d %H:%M:%S') + except ValueError: + abort(400, 'incorrect end date time') + + numfound, total, res = AttributeHistoryManger.get_records(start, end, username, page, page_size) + + return self.jsonify(numfound=numfound, + records=res, + page=page, + total=total, + start=_start, + end=_end, + username=username) + + +class CIHistoryView(APIView): + url_prefix = "/history/ci/" + + def get(self, ci_id): + result = AttributeHistoryManger.get_by_ci_id(ci_id) + return self.jsonify(result) + + +class RecordDetailView(APIView): + url_prefix = "/history/records/" + + def get(self, record_id): + username, timestamp, attr_dict, rel_dict = AttributeHistoryManger.get_record_detail(record_id) + return self.jsonify(username=username, + timestamp=timestamp, + attr_history=attr_dict, + rel_history=rel_dict) diff --git a/api/views/cmdb/preference.py b/api/views/cmdb/preference.py new file mode 100644 index 0000000..d09437f --- /dev/null +++ b/api/views/cmdb/preference.py @@ -0,0 +1,89 @@ +# -*- coding:utf-8 -*- + + +from flask import request + +from api.resource import APIView +from api.lib.perm.acl import has_perm_from_args +from api.lib.cmdb.const import ResourceType, PermEnum +from api.lib.cmdb.preference import PreferenceManager +from api.lib.cmdb.ci import CIManager +from api.lib.cmdb.ci_type import CITypeManager +from api.lib.decorator import args_required +from api.lib.utils import handle_arg_list + + +class PreferenceShowCITypesView(APIView): + url_prefix = "/preference/ci_types" + + def get(self): + instance = request.values.get("instance") + tree = request.values.get("tree") + return self.jsonify(PreferenceManager.get_types(instance, tree)) + + +class PreferenceShowAttributesView(APIView): + url_prefix = "/preference/ci_types//attributes" + + def get(self, id_or_name): + is_subscribed, attributes = PreferenceManager.get_show_attributes(id_or_name) + return self.jsonify(attributes=attributes, is_subscribed=is_subscribed) + + @has_perm_from_args("id_or_name", ResourceType.CI, PermEnum.READ, CITypeManager.get_name_by_id) + @args_required("attr") + def post(self, id_or_name): + id_or_name = int(id_or_name) + attr_list = handle_arg_list(request.values.get("attr", "")) + orders = list(range(len(attr_list))) + PreferenceManager.create_or_update_show_attributes(id_or_name, list(zip(attr_list, orders))) + return self.jsonify(type_id=id_or_name, + attr_order=list(zip(attr_list, orders))) + + @has_perm_from_args("id_or_name", ResourceType.CI, PermEnum.READ, CITypeManager.get_name_by_id) + def put(self, id_or_name): + self.post(id_or_name) + + +class PreferenceTreeApiView(APIView): + url_prefix = "/preference/tree/view" + + def get(self): + return self.jsonify(PreferenceManager.get_tree_view()) + + @has_perm_from_args("type_id", ResourceType.CI, PermEnum.READ, CITypeManager.get_name_by_id) + @args_required("type_id") + @args_required("levels") + def post(self): + type_id = request.values.get("type_id") + levels = handle_arg_list(request.values.get("levels")) + res = PreferenceManager.create_or_update_tree_view(type_id, levels) + return self.jsonify(res and res.to_dict() or {}) + + def put(self): + self.post() + + +class PreferenceRelationApiView(APIView): + url_prefix = "/preference/relation/view" + + def get(self): + return self.jsonify(PreferenceManager.get_relation_view()) + + @has_perm_from_args("parent_id", ResourceType.CI, PermEnum.READ, CITypeManager.get_name_by_id) + @has_perm_from_args("child_id", ResourceType.CI, PermEnum.READ, CITypeManager.get_name_by_id) + @args_required("name") + def post(self): + name = request.values.get("name") + parent_id = request.values.get("parent_id") + child_id = request.values.get("child_id") + res = PreferenceManager.create_or_update_relation_view(name, parent_id, child_id) + return self.jsonify(res.to_dict()) + + def put(self): + self.post() + + @args_required("name") + def delete(self): + name = request.values.get("name") + PreferenceManager.delete_relation_view(name) + return self.jsonify(name=name) diff --git a/api/views/cmdb/relation_type.py b/api/views/cmdb/relation_type.py new file mode 100644 index 0000000..df96302 --- /dev/null +++ b/api/views/cmdb/relation_type.py @@ -0,0 +1,37 @@ +# -*- coding:utf-8 -*- + + +from flask import request +from flask import abort + +from api.resource import APIView +from api.lib.perm.acl import role_required +from api.lib.cmdb.const import RoleEnum +from api.lib.decorator import args_required +from api.lib.cmdb.relation_type import RelationTypeManager + + +class RelationTypeView(APIView): + url_prefix = ("/relation_types", "/relation_types/") + + def get(self): + return self.jsonify([i.to_dict() for i in RelationTypeManager.get_all()]) + + @role_required(RoleEnum.CONFIG) + @args_required("name") + def post(self): + name = request.values.get("name") or abort(400, "Name cannot be empty") + rel = RelationTypeManager.add(name) + return self.jsonify(rel.to_dict()) + + @role_required(RoleEnum.CONFIG) + @args_required("name") + def put(self, rel_id): + name = request.values.get("name") or abort(400, "Name cannot be empty") + rel = RelationTypeManager.update(rel_id, name) + return self.jsonify(rel.to_dict()) + + @role_required(RoleEnum.CONFIG) + def delete(self, rel_id): + RelationTypeManager.delete(rel_id) + return self.jsonify(rel_id=rel_id) diff --git a/api/views/permission.py b/api/views/permission.py new file mode 100644 index 0000000..70cb5b2 --- /dev/null +++ b/api/views/permission.py @@ -0,0 +1,49 @@ +# -*- coding:utf-8 -*- + +from flask import request +from flask import session +from flask_login import current_user + +from api.lib.decorator import args_required +from api.lib.perm.acl import ACLManager +from api.lib.perm.acl import validate_permission +from api.resource import APIView + + +class HasPermissionView(APIView): + url_prefix = "/validate" + + @args_required("resource") + @args_required("resource_type") + @args_required("perm") + def get(self): + resource = request.values.get("resource") + resource_type = request.values.get("resource_type") + perm = request.values.get("perm") + validate_permission(resource, resource_type, perm) + return self.jsonify(is_valid=True) + + def post(self): + self.get() + + +class GetResourcesView(APIView): + url_prefix = "/resources" + + @args_required("resource_type") + def get(self): + resource_type = request.values.get("resource_type") + res = ACLManager().get_resources(resource_type) + return self.jsonify(res) + + +class GetUserInfoView(APIView): + url_prefix = "/user/info" + + def get(self): + name = session.get("acl", {}).get("nickName") or session.get("CAS_USERNAME") or current_user.nickname + role = dict(permissions=session.get("acl", {}).get("parentRoles", []) or ["admin"]) + avatar = session.get("acl", {}).get("avatar") or current_user.avatar + return self.jsonify(result=dict(name=name, + role=role, + avatar=avatar)) diff --git a/autoapp.py b/autoapp.py new file mode 100644 index 0000000..9ff83cc --- /dev/null +++ b/autoapp.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +"""Create an application instance.""" +from flask import g +from flask_login import current_user + +from api.app import create_app + +app = create_app() + + +@app.before_request +def before_request(): + g.user = current_user diff --git a/celery_worker.py b/celery_worker.py new file mode 100644 index 0000000..5692141 --- /dev/null +++ b/celery_worker.py @@ -0,0 +1,9 @@ +# -*- coding:utf-8 -*- + +from api.app import create_app +from api.extensions import celery + +# celery worker -A celery_worker.celery -l DEBUG -E -Q xxxx + +app = create_app() +app.app_context().push() diff --git a/cmdb_api.md b/cmdb_api.md index c4e237f..540e2c3 100644 --- a/cmdb_api.md +++ b/cmdb_api.md @@ -1,13 +1,12 @@ # CMDB API文档 ## 状态返回码的定义 -* 200: 成功 +* 200：成功 * 400：失败 -* 401：未授权 -* 404：url not found -* 408：超时 -* 410：资源删除 -* 500: 服务器错误 +* 401：未认证 +* 403：no permission +* 404：not found +* 500：服务器未知错误 ## 用户接口 diff --git a/command/__init__.py b/command/__init__.py deleted file mode 100644 index 44d37d3..0000000 --- a/command/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding:utf-8 -*- \ No newline at end of file diff --git a/config-sample.cfg b/config-sample.cfg deleted file mode 100644 index 83f4077..0000000 --- a/config-sample.cfg +++ /dev/null @@ -1,61 +0,0 @@ -# coding: utf-8 -# common - -DEBUG = True -SECRET_KEY = 'dsfdjsf@3213!@JKJWL' -HOST = 'http://127.0.0.1:5000' - -# # database -SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://mysqluser:password@127.0.0.1:3306/cmdb?charset=utf8' - -SQLALCHEMY_ECHO = False -SQLALCHEMY_POOL_SIZE = 10 -SQLALCHEMY_POOL_RECYCLE = 300 - -# # cache -CACHE_TYPE = "redis" -CACHE_REDIS_HOST = "127.0.0.1" -CACHE_REDIS_PORT = 6379 -CACHE_KEY_PREFIX = "CMDB-API" -CACHE_DEFAULT_TIMEOUT = 3000 - - -# # CI cache -REDIS_DB = 0 -REDIS_MAX_CONN = 30 - - -# # queue -CELERY_RESULT_BACKEND = "redis://127.0.0.1//" -BROKER_URL = 'redis://127.0.0.1//' -BROKER_VHOST = '/' - - -# # i18n -ACCEPT_LANGUAGES = ['en', 'zh'] -BABEL_DEFAULT_LOCALE = 'zh' -BABEL_DEFAULT_TIMEZONE = 'Asia/Shanghai' - -# # log -LOG_PATH = './logs/app.log' - -LOG_LEVEL = 'DEBUG' -ADMINS = ('@') - -# # mail -MAIL_SERVER = '' -MAIL_PORT = 25 -MAIL_USE_TLS = False -MAIL_USE_SSL = False -MAIL_DEBUG = True -MAIL_USERNAME = '' -MAIL_PASSWORD = '' -DEFAULT_MAIL_SENDER = '' - - -# # pagination -PER_PAGE_COUNT_RANGE = (10, 25, 50, 100) -DEFAULT_PAGE_COUNT = 25 - - -WHITE_LIST = ["127.0.0.1"] diff --git a/core/__init__.py b/core/__init__.py deleted file mode 100644 index 218d127..0000000 --- a/core/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding:utf-8 -*- - - -from attribute import attribute -from ci_type import citype -from ci_type_relation import cityperelation -from ci_relation import cirelation -from ci import ci -from history import history -from account import account -from special import special diff --git a/core/account.py b/core/account.py deleted file mode 100644 index 1a1ae82..0000000 --- a/core/account.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import Blueprint -from flask import request - -from flask import g -from flask import abort -from flask import jsonify - -from models import row2dict -from lib.account import AccountManager -from lib.auth import auth_with_key - - -account = Blueprint('account', __name__) - - -@account.route("/", methods=["GET"]) -@auth_with_key -def get_user(uid=None): - manager = AccountManager() - user = manager.get_user_by_uid(uid) - if user: - return jsonify(rolenames=user.rolenames, user=row2dict(user)) - else: - return jsonify(user=None) - - -@account.route("", methods=["POST"]) -@auth_with_key -def create_user(): - manager = AccountManager() - params = {} - for k, v in request.values.iteritems(): - params[k] = v - user = manager.create_user(**params) - return jsonify(user=row2dict(user)) - - -@account.route("/", methods=["PUT"]) -@auth_with_key -def update_user(uid=None): - manager = AccountManager() - params = {} - for k, v in request.values.iteritems(): - params[k] = v - ret, res = manager.update_user(uid, **params) - if not ret: - abort(res[0], res[1]) - return jsonify(user=row2dict(res), rolenames=res.rolenames) - - -@account.route("/", methods=["DELETE"]) -@auth_with_key -def delete_user(uid=None): - manager = AccountManager() - ret, res = manager.delete_user(uid) - if not ret: - abort(res[0], res[1]) - return jsonify(uid=uid) - - -@account.route("/validate", methods=["POST"]) -@auth_with_key -def validate(): - username = request.values.get("username") - password = request.values.get("password") - manager = AccountManager() - user, authenticated = manager.validate(username, password) - if user and not authenticated: - return jsonify(code=401, user=row2dict(user), rolenames=user.rolenames) - elif not user: - return jsonify(code=404, message="user is not existed") - return jsonify(code=200, user=row2dict(user), rolenames=user.rolenames) - - -@account.route("/key", methods=["PUT"]) -@auth_with_key -def update_key(): - manager = AccountManager() - ret, res = manager.reset_key(g.user.uid) - if not ret: - abort(res[0], res[1]) - return jsonify(user=row2dict(res), rolenames=res.rolenames) - - -@account.route("/password", methods=["PUT"]) -@auth_with_key -def update_password(): - manager = AccountManager() - old = request.values.get("password") - new = request.values.get("new_password") - confirm = request.values.get("confirm") - ret, res = manager.update_password(g.user.uid, old, new, confirm) - if not ret: - abort(res[0], res[1]) - return jsonify(user=row2dict(res), rolenames=res.rolenames) diff --git a/core/attribute.py b/core/attribute.py deleted file mode 100644 index 1d79a7b..0000000 --- a/core/attribute.py +++ /dev/null @@ -1,152 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import jsonify -from flask import request -from flask import Blueprint -from flask import abort -from flask import current_app - -from lib.attribute import AttributeManager -from lib.ci_type import CITypeAttributeManager -from lib.decorator import argument_required -from lib.exception import InvalidUsageError -from lib.auth import auth_with_key - -attribute = Blueprint("attribute", __name__) - - -@attribute.route("", methods=["GET"]) -def get_attributes(): - q = request.values.get("q") - attrs = AttributeManager().get_attributes(name=q) - count = len(attrs) - return jsonify(numfound=count, attributes=attrs) - - -@attribute.route("/", methods=["GET"]) -@attribute.route("/", methods=["GET"]) -def get_attribute(attr_name=None, attr_id=None): - attr_manager = AttributeManager() - attr_dict = None - if attr_name is not None: - attr_dict = attr_manager.get_attribute_by_name(attr_name) - if attr_dict is None: - attr_dict = attr_manager.get_attribute_by_alias(attr_name) - elif attr_id is not None: - attr_dict = attr_manager.get_attribute_by_id(attr_id) - if attr_dict is not None: - return jsonify(attribute=attr_dict) - abort(404, "attribute not found") - - -@attribute.route("", methods=["POST"]) -@auth_with_key -def create_attribute(): - with argument_required("attr_name"): - attr_name = request.values.get("attr_name") - current_app.logger.info(attr_name) - attr_alias = request.values.get("attr_alias", attr_name) - choice_value = request.values.get("choice_value") - is_multivalue = request.values.get("is_multivalue", False) - is_uniq = request.values.get("is_uniq", False) - is_index = request.values.get("is_index", False) - value_type = request.values.get("value_type", "text") - try: - is_multivalue = int(is_multivalue) - is_uniq = int(is_uniq) - is_index = int(is_index) - except ValueError: - raise InvalidUsageError("argument format is error") - attr_manager = AttributeManager() - kwargs = {"choice_value": choice_value, "is_multivalue": is_multivalue, - "is_uniq": is_uniq, "value_type": value_type, - "is_index": is_index} - ret, res = attr_manager.add(attr_name, attr_alias, **kwargs) - if not ret: - return abort(500, res) - return jsonify(attr_id=res) - - -@attribute.route("/", methods=["PUT"]) -@auth_with_key -def update_attribute(attr_id=None): - with argument_required("attr_name"): - attr_name = request.values.get("attr_name") - attr_alias = request.values.get("attr_alias", attr_name) - choice_value = request.values.get("choice_value") - is_multivalue = request.values.get("is_multivalue", False) - is_uniq = request.values.get("is_uniq", False) - value_type = request.values.get("value_type", "text") - try: - is_multivalue = int(is_multivalue) - is_uniq = int(is_uniq) - except ValueError: - raise InvalidUsageError("argument format is error") - attr_manager = AttributeManager() - kwargs = {"choice_value": choice_value, "is_multivalue": is_multivalue, - "is_uniq": is_uniq, "value_type": value_type} - ret, res = attr_manager.update(attr_id, attr_name, - attr_alias, **kwargs) - if not ret: - return abort(500, res) - return jsonify(attr_id=res) - - -@attribute.route("/", methods=["DELETE"]) -@auth_with_key -def delete_attribute(attr_id=None): - attr_manager = AttributeManager() - res = attr_manager.delete(attr_id) - return jsonify(message="attribute {0} deleted".format(res)) - - -@attribute.route("/citype/", methods=["GET"]) -@attribute.route("/citype/", methods=["GET"]) -def get_attributes_by_type(type_id=None, type_name=None): - manager = CITypeAttributeManager() - from models.attribute import CIAttributeCache - from models.ci_type import CITypeCache - from models.ci_type import CITypeAttributeCache - - t = CITypeCache.get(type_id) - if not t: - t = CITypeCache.get(type_name) - if not t: - return abort(400, "CIType {0} is not existed".format(type_id)) - type_id = t.type_id - uniq_id = t.uniq_id - CITypeAttributeCache.clean(type_id) - unique = CIAttributeCache.get(uniq_id).attr_name - return jsonify(attributes=manager.get_attributes_by_type_id(type_id), - type_id=type_id, uniq_id=uniq_id, unique=unique) - - -@attribute.route("/citype/", methods=["POST"]) -@auth_with_key -def create_attributes_to_citype(type_id=None): - with argument_required("attr_id"): - attr_ids = request.values.get("attr_id", "") - is_required = request.values.get("is_required", False) - attr_id_list = attr_ids.strip().split(",") - if "" in attr_id_list: - attr_id_list.remove("") - attr_id_list = map(int, attr_id_list) - try: - is_required = int(is_required) - except ValueError: - abort(500, "argument format is error") - manager = CITypeAttributeManager() - manager.add(type_id, attr_id_list, is_required=is_required) - return jsonify(attributes=attr_id_list) - - -@attribute.route("/citype/", methods=["DELETE"]) -@auth_with_key -def delete_attribute_in_type(type_id=None): - with argument_required("attr_id"): - attr_ids = request.values.get("attr_id", "") - attr_id_list = attr_ids.strip().split(",") - manager = CITypeAttributeManager() - manager.delete(type_id, attr_id_list) - return jsonify(attributes=attr_id_list) \ No newline at end of file diff --git a/core/ci.py b/core/ci.py deleted file mode 100644 index 3f13eae..0000000 --- a/core/ci.py +++ /dev/null @@ -1,216 +0,0 @@ -# -*- coding:utf-8 -*- - -import sys -reload(sys) -sys.setdefaultencoding("utf-8") -import time -import urllib - -from flask import Blueprint -from flask import request -from flask import jsonify -from flask import current_app -from flask import make_response -from flask import render_template -from flask import abort - -from lib.auth import auth_with_key -from lib.ci import CIManager -from lib.ci import HostNumStatis -from lib.search import Search -from lib.search import SearchError -from lib.utils import get_page -from lib.utils import get_per_page -from models.ci_type import CITypeCache - -ci = Blueprint("ci", __name__) - - -@ci.route("/type/", methods=["GET"]) -def get_cis_by_type(type_id=None): - fields = request.args.get("fields", "").strip().split(",") - fields = filter(lambda x: x != "", fields) - - ret_key = request.args.get("ret_key", "name") - if ret_key not in ('name', 'alias', 'id'): - ret_key = 'name' - - page = get_page(request.values.get("page", 1)) - count = get_per_page(request.values.get("count")) - manager = CIManager() - res = manager.get_cis_by_type(type_id, ret_key=ret_key, - fields=fields, page=page, per_page=count) - return jsonify(type_id=type_id, numfound=res[0], - total=len(res[2]), page=res[1], cis=res[2]) - - -@ci.route("/", methods=['GET']) -def get_ci(ci_id=None): - fields = request.args.get("fields", "").strip().split(",") - fields = filter(lambda x: x != "", fields) - - ret_key = request.args.get("ret_key", "name") - if ret_key not in ('name', 'alias', 'id'): - ret_key = 'name' - - manager = CIManager() - ci = manager.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) - return jsonify(ci_id=ci_id, ci=ci) - - -@ci.route("/s", methods=["GET"]) -@ci.route("/search", methods=["GET"]) -def search(): - """@params: q: query statement - fl: filter by column - count: the number of ci - ret_key: id, name, alias - facet: statistic - wt: result format - """ - page = get_page(request.values.get("page", 1)) - count = get_per_page(request.values.get("count")) - - query = request.values.get('q', "") - fl = request.values.get('fl', "").split(",") - ret_key = request.values.get('ret_key', "name") - if ret_key not in ('name', 'alias', 'id'): - ret_key = 'name' - facet = request.values.get("facet", "").split(",") - wt = request.values.get('wt', 'json') - fl = filter(lambda x: x != "", fl) - facet = filter(lambda x: x != "", facet) - sort = request.values.get("sort") - - start = time.time() - s = Search(query, fl, facet, page, ret_key, count, sort) - try: - response, counter, total, page, numfound, facet = s.search() - except SearchError, e: - return abort(400, str(e)) - except Exception, e: - current_app.logger.error(str(e)) - return abort(500, "search unknown error") - - if wt == 'xml': - res = make_response( - render_template("search.xml", - counter=counter, - total=total, - result=response, - page=page, - numfound=numfound, - facet=facet)) - res.headers['Content-type'] = 'text/xml' - return res - current_app.logger.debug("search time is :{0}".format( - time.time() - start)) - return jsonify(numfound=numfound, - total=total, - page=page, - facet=facet, - counter=counter, - result=response) - - -@ci.route("", methods=["POST"]) -@auth_with_key -def create_ci(): - ci_type = request.values.get("ci_type") - _no_attribute_policy = request.values.get("_no_attribute_policy", "ignore") - - ci_dict = dict() - for k, v in request.values.iteritems(): - if k != "ci_type" and not k.startswith("_"): - ci_dict[k] = v.strip() - - manager = CIManager() - current_app.logger.debug(ci_dict) - ci_id = manager.add(ci_type, exist_policy="reject", - _no_attribute_policy=_no_attribute_policy, **ci_dict) - return jsonify(ci_id=ci_id) - - -@ci.route("", methods=["PUT"]) -@auth_with_key -def update_ci(): - if request.data: - args = dict() - _args = request.data.split("&") - for arg in _args: - if arg: - args[arg.split("=")[0]] = \ - urllib.unquote(urllib.unquote(arg.split("=")[1])) - else: - args = request.values - - ci_type = args.get("ci_type") - _no_attribute_policy = args.get("_no_attribute_policy", "ignore") - ci_dict = dict() - for k, v in args.items(): - if k != "ci_type" and not k.startswith("_"): - ci_dict[k] = v.strip() - - manager = CIManager() - ci_id = manager.add(ci_type, exist_policy="replace", - _no_attribute_policy=_no_attribute_policy, **ci_dict) - return jsonify(ci_id=ci_id) - - -@ci.route("//unique", methods=["PUT"]) -@auth_with_key -def update_ci_unique(ci_id): - m = CIManager() - m.update_unique_value(ci_id, request.values) - return jsonify(ci_id=ci_id) - - -@ci.route("/", methods=["DELETE"]) -@auth_with_key -def delete_ci(ci_id=None): - manager = CIManager() - manager.delete(ci_id) - return jsonify(message="ok") - - -@ci.route("/heartbeat//", methods=["POST"]) -def add_heartbeat(ci_type, unique): - if not unique or not ci_type: - return jsonify(message="error") - # return jsonify(message="ok") - return jsonify(message=CIManager().add_heartbeat(ci_type, unique)) - - -@ci.route("/heartbeat", methods=["GET"]) -def get_heartbeat(): - page = get_page(request.values.get("page", 1)) - ci_type = request.values.get("ci_type", "").strip() - try: - ci_type = CITypeCache.get(ci_type).type_id - except: - return jsonify(numfound=0, result=[]) - agent_status = request.values.get("agent_status", None) - if agent_status: - agent_status = int(agent_status) - numfound, result = CIManager().get_heartbeat(page, - ci_type, - agent_status=agent_status) - return jsonify(numfound=numfound, result=result) - - -######################### just for frontend ################################### - -@ci.route("/hosts/nums", methods=["GET"]) -def get_hosts_nums(): - ci_type = request.args.get("ci_type", "").strip() - ci_ids = request.args.get("ci_ids", "").strip() - ci_id_list = ci_ids.split(",") - ci_id_list = map(str, filter(lambda x: x != "", ci_id_list)) - res = {} - if ci_type == "bu": - res = HostNumStatis().get_hosts_by_bu(ci_id_list) - elif ci_type == "product": - res = HostNumStatis().get_hosts_by_product(ci_id_list) - elif ci_type == "project": - res = HostNumStatis().get_hosts_by_project(ci_id_list) - return jsonify(hosts=res) \ No newline at end of file diff --git a/core/ci_relation.py b/core/ci_relation.py deleted file mode 100644 index 63233df..0000000 --- a/core/ci_relation.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import Blueprint -from flask import jsonify -from flask import request - -from lib.ci import CIRelationManager -from lib.utils import get_page -from lib.utils import get_per_page -from lib.auth import auth_with_key - - -cirelation = Blueprint("cirelation", __name__) - - -@cirelation.route("/types", methods=["GET"]) -def get_types(): - manager = CIRelationManager() - return jsonify(relation_types=manager.relation_types) - - -@cirelation.route("//second_cis", methods=["GET"]) -def get_second_cis_by_first_ci(first_ci=None): - page = get_page(request.values.get("page", 1)) - count = get_per_page(request.values.get("count")) - relation_type = request.values.get("relation_type", "contain") - manager = CIRelationManager() - numfound, total, second_cis = manager.get_second_cis( - first_ci, page=page, per_page=count, relation_type=relation_type) - return jsonify(numfound=numfound, total=total, - page=page, second_cis=second_cis) - - -@cirelation.route("//first_cis", methods=["GET"]) -def get_first_cis_by_second_ci(second_ci=None): - page = get_page(request.values.get("page", 1)) - count = get_per_page(request.values.get("count")) - relation_type = request.values.get("relation_type", "contain") - - manager = CIRelationManager() - numfound, total, first_cis = manager.get_first_cis( - second_ci, per_page=count, page=page, relation_type=relation_type) - return jsonify(numfound=numfound, total=total, - page=page, first_cis=first_cis) - - -@cirelation.route("//", methods=["POST"]) -@auth_with_key -def create_ci_relation(first_ci=None, second_ci=None): - relation_type = request.values.get("relation_type", "contain") - manager = CIRelationManager() - res = manager.add(first_ci, second_ci, relation_type=relation_type) - return jsonify(cr_id=res) - - -@cirelation.route("/", methods=["DELETE"]) -@auth_with_key -def delete_ci_relation(cr_id=None): - manager = CIRelationManager() - manager.delete(cr_id) - return jsonify(message="CIType Relation is deleted") - - -@cirelation.route("//", methods=["DELETE"]) -@auth_with_key -def delete_ci_relation_2(first_ci, second_ci): - manager = CIRelationManager() - manager.delete_2(first_ci, second_ci) - return jsonify(message="CIType Relation is deleted") \ No newline at end of file diff --git a/core/ci_type.py b/core/ci_type.py deleted file mode 100644 index 18aa23c..0000000 --- a/core/ci_type.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import Blueprint -from flask import jsonify -from flask import request -from flask import abort - -from lib.ci_type import CITypeManager -from lib.decorator import argument_required -from lib.auth import auth_with_key - - -citype = Blueprint("citype", __name__) - - -@citype.route("", methods=["GET"]) -def get_citypes(): - type_name = request.args.get("type_name") - manager = CITypeManager() - citypes = manager.get_citypes(type_name) - count = len(citypes) - return jsonify(numfound=count, citypes=citypes) - - -@citype.route("/query", methods=["GET"]) -def query(): - with argument_required("type"): - _type = request.args.get("type") - manager = CITypeManager() - res = manager.query(_type) - return jsonify(citype=res) - - -@citype.route("", methods=["POST"]) -@auth_with_key -def create_citype(): - with argument_required("type_name"): - type_name = request.values.get("type_name") - type_alias = request.values.get("type_alias") - if type_alias is None: - type_alias = type_name - _id = request.values.get("_id") - unique = request.values.get("unique") - enabled = request.values.get("enabled", True) - icon_url = request.values.get("icon_url", "") - manager = CITypeManager() - ret, res = manager.add(type_name, type_alias, _id=_id, - unique=unique, enabled=enabled, - icon_url=icon_url) - if ret: - return jsonify(type_id=res) - abort(500, res) - - -@citype.route("/", methods=["PUT"]) -@auth_with_key -def update_citype(type_id=None): - type_name = request.values.get("type_name") - type_alias = request.values.get("type_alias") - _id = request.values.get("_id") - unique = request.values.get("unique") - icon_url = request.values.get("icon_url") - enabled = request.values.get("enabled") - enabled = False if enabled in (0, "0") else True \ - if enabled is not None else None - manager = CITypeManager() - ret, res = manager.update(type_id, type_name, type_alias, _id=_id, - unique=unique, icon_url=icon_url, - enabled=enabled) - if ret: - return jsonify(type_id=type_id) - abort(500, res) - - -@citype.route("/", methods=["DELETE"]) -@auth_with_key -def delete_citype(type_id=None): - manager = CITypeManager() - res = manager.delete(type_id) - return jsonify(message=res) - - -@citype.route("/enable/", methods=["GET", "POST"]) -def enable(type_id=None): - enable = request.values.get("enable", True) - manager = CITypeManager() - manager.set_enabled(type_id, enabled=enable) - return jsonify(type_id=type_id) diff --git a/core/ci_type_relation.py b/core/ci_type_relation.py deleted file mode 100644 index 36d72ca..0000000 --- a/core/ci_type_relation.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import Blueprint -from flask import jsonify -from flask import request - -from lib.ci_type import CITypeRelationManager -from lib.auth import auth_with_key - - -cityperelation = Blueprint("cityperelation", __name__) - - -@cityperelation.route("/types", methods=["GET"]) -def get_types(): - manager = CITypeRelationManager() - return jsonify(relation_types=manager.relation_types) - - -@cityperelation.route("//children", methods=["GET"]) -def get_children_by_parent(parent=None): - manager = CITypeRelationManager() - return jsonify(children=manager.get_children(parent)) - - -@cityperelation.route("//parents", methods=["GET"]) -def get_parents_by_child(child=None): - manager = CITypeRelationManager() - return jsonify(parents=manager.get_parents(child)) - - -@cityperelation.route("//", methods=["POST"]) -@auth_with_key -def create_citype_realtions(parent=None, child=None): - relation_type = request.values.get("relation_type", "contain") - manager = CITypeRelationManager() - res = manager.add(parent, child, relation_type=relation_type) - return jsonify(ctr_id=res) - - -@cityperelation.route("/", methods=["DELETE"]) -@auth_with_key -def delete_citype_relation(ctr_id=None): - manager = CITypeRelationManager() - manager.delete(ctr_id) - return jsonify(message="CIType Relation is deleted") - - -@cityperelation.route("//", methods=["DELETE"]) -@auth_with_key -def delete_citype_relation_2(parent=None, child=None): - manager = CITypeRelationManager() - manager.delete_2(parent, child) - return jsonify(message="CIType Relation is deleted") diff --git a/core/history.py b/core/history.py deleted file mode 100644 index 5eee828..0000000 --- a/core/history.py +++ /dev/null @@ -1,116 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from flask import jsonify -from flask import current_app -from flask import Blueprint -from flask import request -from flask import abort - -from models.history import OperationRecord -from models.history import CIRelationHistory -from models.history import CIAttributeHistory -from models.attribute import CIAttributeCache -from extensions import db -from models import row2dict -from models.account import UserCache -from lib.ci import CIManager -from lib.utils import get_page - -history = Blueprint("history", __name__) - - -@history.route("/record", methods=["GET"]) -def get_record(): - page = get_page(request.values.get("page", 1)) - _start = request.values.get("start") - _end = request.values.get("end") - username = request.values.get("username", "") - per_page_cnt = current_app.config.get("DEFAULT_PAGE_COUNT") - start, end = None, None - if _start: - try: - start = datetime.datetime.strptime(_start, '%Y-%m-%d %H:%M:%S') - except ValueError: - abort(400, 'incorrect start date time') - if _end: - try: - end = datetime.datetime.strptime(_end, '%Y-%m-%d %H:%M:%S') - except ValueError: - abort(400, 'incorrect end date time') - records = db.session.query(OperationRecord) - numfound = db.session.query(db.func.count(OperationRecord.record_id)) - if start: - records = records.filter(OperationRecord.timestamp >= start) - numfound = numfound.filter(OperationRecord.timestamp >= start) - if end: - records = records.filter(OperationRecord.timestamp <= end) - numfound = records.filter(OperationRecord.timestamp <= end) - if username: - user = UserCache.get(username) - if user: - records = records.filter(OperationRecord.uid == user.uid) - else: - return jsonify(numfound=0, records=[], - page=1, total=0, start=_start, - end=_end, username=username) - records = records.order_by(-OperationRecord.record_id).offset( - per_page_cnt * (page - 1)).limit(per_page_cnt).all() - total = len(records) - numfound = numfound.first()[0] - res = [] - for record in records: - _res = row2dict(record) - _res["user"] = UserCache.get(_res.get("uid")).nickname \ - if UserCache.get(_res.get("uid")).nickname \ - else UserCache.get(_res.get("uid")).username - attr_history = db.session.query(CIAttributeHistory.attr_id).filter( - CIAttributeHistory.record_id == _res.get("record_id")).all() - _res["attr_history"] = [CIAttributeCache.get(h.attr_id).attr_alias - for h in attr_history] - rel_history = db.session.query(CIRelationHistory.operate_type).filter( - CIRelationHistory.record_id == _res.get("record_id")).all() - rel_statis = {} - for rel in rel_history: - if rel.operate_type not in rel_statis: - rel_statis[rel.operate_type] = 1 - else: - rel_statis[rel.res.operate_type] += 1 - _res["rel_history"] = rel_statis - res.append(_res) - - return jsonify(numfound=numfound, records=res, page=page, total=total, - start=_start, end=_end, username=username) - - -@history.route("/", methods=["GET"]) -def get_detail_by_record(record_id=None): - record = db.session.query(OperationRecord).filter( - OperationRecord.record_id == record_id).first() - if record is None: - abort(404, "record is not found") - username = UserCache.get(record.uid).nickname \ - if UserCache.get(record.uid).nickname \ - else UserCache.get(record.uid).username - timestamp = record.timestamp.strftime("%Y-%m-%d %H:%M:%S") - attr_history = db.session.query(CIAttributeHistory).filter( - CIAttributeHistory.record_id == record_id).all() - rel_history = db.session.query(CIRelationHistory).filter( - CIRelationHistory.record_id == record_id).all() - attr_dict, rel_dict = dict(), {"add": [], "delete": []} - for attr_h in attr_history: - attr_dict[CIAttributeCache.get(attr_h.attr_id).attr_alias] = { - "old": attr_h.old, "new": attr_h.new, - "operate_type": attr_h.operate_type} - manager = CIManager() - for rel_h in rel_history: - _, first = manager.get_ci_by_id(rel_h.first_ci_id) - _, second = manager.get_ci_by_id(rel_h.second_ci_id) - rel_dict[rel_h.operate_type].append( - (first, rel_h.relation_type, second)) - - return jsonify(username=username, timestamp=timestamp, - attr_history=attr_dict, - rel_history=rel_dict) diff --git a/core/statis.py b/core/statis.py deleted file mode 100644 index 20b5061..0000000 --- a/core/statis.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import Blueprint - - -statis = Blueprint("statis", __name__) - - -@statis.route("") -def statis(): - pass \ No newline at end of file diff --git a/extensions.py b/extensions.py deleted file mode 100644 index 176c96c..0000000 --- a/extensions.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - - -from flask.ext.mail import Mail -from flask.ext.sqlalchemy import SQLAlchemy -from flask.ext.cache import Cache -from flask.ext.celery import Celery - -from lib.utils import RedisHandler - - -__all__ = ['mail', 'db', 'cache', 'celery', "rd"] - - -mail = Mail() -db = SQLAlchemy() -cache = Cache() -celery = Celery() -rd = RedisHandler() \ No newline at end of file diff --git a/gunicornserver.py b/gunicornserver.py deleted file mode 100644 index 996804e..0000000 --- a/gunicornserver.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*- - -from flask_script import Command, Option - - -class GunicornServer(Command): - description = 'Run the app within Gunicorn' - - def __init__(self, host='127.0.0.1', port=5000, workers=8, - worker_class="sync", daemon=False): - self.port = port - self.host = host - self.workers = workers - self.worker_class = worker_class - self.daemon = daemon - - def get_options(self): - return ( - Option('-H', '--host', - dest='host', - default=self.host), - - Option('-p', '--port', - dest='port', - type=int, - default=self.port), - - Option('-w', '--workers', - dest='workers', - type=int, - default=self.workers), - - Option("-c", "--worker_class", - dest='worker_class', - type=str, - default=self.worker_class), - - Option("-d", "--daemon", - dest="daemon", - type=bool, - default=self.daemon) - ) - - def handle(self, app, host, port, workers, worker_class, daemon): - - from gunicorn import version_info - - if version_info < (0, 9, 0): - from gunicorn.arbiter import Arbiter - from gunicorn.config import Config - - arbiter = Arbiter(Config({'bind': "%s:%d" % (host, int(port)), - 'workers': workers, - 'worker_class': worker_class, - 'daemon': daemon}), app) - arbiter.run() - else: - from gunicorn.app.base import Application - - class FlaskApplication(Application): - def init(self, parser, opts, args): - return { - 'bind': '{0}:{1}'.format(host, port), - 'workers': workers, - 'worker_class': worker_class, - 'daemon': daemon - } - - def load(self): - return app - - FlaskApplication().run() diff --git a/lib/account.py b/lib/account.py deleted file mode 100644 index 31bb2b1..0000000 --- a/lib/account.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding:utf-8 -*- - - -import uuid -import random -import string -import datetime - -from flask import current_app -from flask import abort - -from extensions import db -from models.account import UserCache -from models.account import User -from models.account import UserRole - - -class AccountManager(object): - def __init__(self): - pass - - def get_user_by_uid(self, uid): - user = UserCache.get(uid) - return user - - def _generate_key(self): - key = uuid.uuid4().hex - secret = ''.join(random.sample(string.ascii_letters + - string.digits + '~!@#$%^&*?', 32)) - return key, secret - - def validate(self, username, password): - user, authenticated = User.query.authenticate(username, password) - return user, authenticated - - def create_user(self, **kwargs): - username = kwargs.get("username") - if username: - user = UserCache.get(username) - if user is not None: - user, authenticated = self.validate( - username, kwargs.get("password")) - if authenticated: - return user - else: - return abort(401, "authenticate validate failed") - else: - return abort(400, "argument username is required") - user = User() - email = kwargs.get("email", "") - if not email: - return abort(400, "argument email is required") - user.email = email - user.password = kwargs.get("password") - user.username = kwargs.get("username", "") - user.nickname = kwargs.get("nickname") if kwargs.get("nickname") \ - else kwargs.get("username", "") - key, secret = self._generate_key() - user.key = key - user.secret = secret - user.date_joined = datetime.datetime.now() - user.block = 0 - - db.session.add(user) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("create user is error {0}".format(str(e))) - return abort(500, "create user is error, {0}".format(str(e))) - return user - - def update_user(self, uid, **kwargs): - user = UserCache.get(uid) - if user is None: - return abort(400, "the user[{0}] is not existed".format(uid)) - user.username = kwargs.get("username", "") \ - if kwargs.get("username") else user.username - user.nickname = kwargs.get("nickname") \ - if kwargs.get("nickname") else user.nickname - user.department = kwargs.get("department") \ - if kwargs.get("department") else user.department - user.catalog = kwargs.get("catalog") \ - if kwargs.get("catalog") else user.catalog - user.email = kwargs.get("email") \ - if kwargs.get("email") else user.email - user.mobile = kwargs.get("mobile") \ - if kwargs.get("mobile") else user.mobile - db.session.add(user) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("create user is error {0}".format(str(e))) - return abort(500, "create user is error, {0}".format(str(e))) - return True, user - - def delete_user(self, uid): - user = UserCache.get(uid) - if user is None: - return abort(400, "the user[{0}] is not existed".format(uid)) - db.session.query(UserRole).filter(UserRole.uid == uid).delete() - db.session.delete(user) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("delete user error, {0}".format(str(e))) - return abort(500, "delete user error, {0}".format(str(e))) - return True, uid - - def update_password(self, uid, old, new, confirm): - user = User.query.get(uid) - if not user: - return abort(400, "user is not existed") - if not user.check_password(old): - return abort(400, "invalidate old password") - if not (new and confirm and new == confirm): - return abort(400, """Password cannot be empty, - two inputs must be the same""") - user.password = new - db.session.add(user) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("set password error, %s" % str(e)) - return abort(500, "set password errors, {0:s}".format(str(e))) - return True, user - - def reset_key(self, uid): - user = UserCache.get(uid) - if user is None: - return abort(400, "the user[{0}] is not existed".format(uid)) - key, secret = self._generate_key() - user.key = key - user.secret = secret - db.session.add(user) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("reset key is error, {0}".format(str(e))) - return abort(500, "reset key is error, {0}".format(str(e))) - return True, user \ No newline at end of file diff --git a/lib/attribute.py b/lib/attribute.py deleted file mode 100644 index dd6f6ca..0000000 --- a/lib/attribute.py +++ /dev/null @@ -1,168 +0,0 @@ -# -*- coding:utf-8 -*- - -from flask import current_app -from flask import abort - -from extensions import db -from models.attribute import CIAttribute -from models.attribute import CIAttributeCache -from models import row2dict -from lib.const import type_map - - -class AttributeManager(object): - """ - CI attributes manager - """ - - def __init__(self): - pass - - def _get_choice_value(self, attr_id, value_type): - _table = type_map.get("choice").get(value_type) - choice_values = db.session.query(_table.value).filter( - _table.attr_id == attr_id).all() - return [choice_value.value for choice_value in choice_values] - - def _add_choice_value(self, choice_value, attr_id, value_type): - _table = type_map.get("choice").get(value_type) - db.session.query(_table).filter(_table.attr_id == attr_id).delete() - db.session.flush() - for v in choice_value.strip().split(","): - table = _table() - table.attr_id = attr_id - table.value = v - db.session.add(table) - db.session.flush() - - def get_attributes(self, name=None): - """ - return attribute by name, - if name is None, then return all attributes - """ - attrs = db.session.query(CIAttribute).filter( - CIAttribute.attr_name.ilike("%{0}%".format(name))).all() \ - if name is not None else db.session.query(CIAttribute).all() - res = list() - for attr in attrs: - attr_dict = row2dict(attr) - if attr.is_choice: - attr_dict["choice_value"] = self._get_choice_value( - attr.attr_id, attr.value_type) - res.append(attr_dict) - return res - - def get_attribute_by_name(self, attr_name): - attr = db.session.query(CIAttribute).filter( - CIAttribute.attr_name == attr_name).first() - if attr: - attr_dict = row2dict(attr) - if attr.is_choice: - attr_dict["choice_value"] = self._get_choice_value( - attr.attr_id, attr.value_type) - return attr_dict - - def get_attribute_by_alias(self, attr_alias): - attr = db.session.query(CIAttribute).filter( - CIAttribute.attr_alias == attr_alias).first() - if attr: - attr_dict = row2dict(attr) - if attr.is_choice: - attr_dict["choice_value"] = self._get_choice_value( - attr.attr_id, attr.value_type) - return attr_dict - - def get_attribute_by_id(self, attr_id): - attr = db.session.query(CIAttribute).filter( - CIAttribute.attr_id == attr_id).first() - if attr: - attr_dict = row2dict(attr) - if attr.is_choice: - attr_dict["choice_value"] = self._get_choice_value( - attr.attr_id, attr.value_type) - return attr_dict - - def add(self, attr_name, attr_alias, **kwargs): - choice_value = kwargs.get("choice_value", False) - attr = CIAttributeCache.get(attr_name) - if attr is not None: - return False, "attribute {0} is already existed".format(attr_name) - is_choice = False - if choice_value: - is_choice = True - if not attr_alias: - attr_alias = attr_name - attr = CIAttribute() - attr.attr_name = attr_name - attr.attr_alias = attr_alias - attr.is_choice = is_choice - attr.is_multivalue = kwargs.get("is_multivalue", False) - attr.is_uniq = kwargs.get("is_uniq", False) - attr.is_index = kwargs.get("is_index", False) - attr.value_type = kwargs.get("value_type", "text") - db.session.add(attr) - db.session.flush() - - if choice_value: - self._add_choice_value(choice_value, attr.attr_id, attr.value_type) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("add attribute error, {0}".format(str(e))) - return False, str(e) - CIAttributeCache.clean(attr) - return True, attr.attr_id - - def update(self, attr_id, *args, **kwargs): - attr = db.session.query(CIAttribute).filter_by(attr_id=attr_id).first() - if not attr: - return False, "CI attribute you want to update is not existed" - choice_value = kwargs.get("choice_value", False) - is_choice = False - if choice_value: - is_choice = True - attr.attr_name = args[0] - attr.attr_alias = args[1] - if not args[1]: - attr.attr_alias = args[0] - attr.is_choice = is_choice - attr.is_multivalue = kwargs.get("is_multivalue", False) - attr.is_uniq = kwargs.get("is_uniq", False) - attr.value_type = kwargs.get("value_type", "text") - db.session.add(attr) - db.session.flush() - if is_choice: - self._add_choice_value(choice_value, attr.attr_id, attr.value_type) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("update attribute error, {0}".format( - str(e))) - return False, str(e) - CIAttributeCache.clean(attr) - return True, attr.attr_id - - def delete(self, attr_id): - attr, name = db.session.query(CIAttribute).filter_by( - attr_id=attr_id).first(), None - if attr: - if attr.is_choice: - choice_table = type_map["choice"].get(attr.value_type) - db.session.query(choice_table).filter( - choice_table.attr_id == attr_id).delete() - db.session.flush() - name = attr.attr_name - CIAttributeCache.clean(attr) - db.session.delete(attr) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("delete attribute error, {0}".format( - str(e))) - return abort(500, str(e)) - else: - return abort(404, "attribute you want to delete is not existed") - return name \ No newline at end of file diff --git a/lib/auth.py b/lib/auth.py deleted file mode 100644 index 7147e78..0000000 --- a/lib/auth.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding:utf-8 -*- - -import urllib -from functools import wraps - -from flask import current_app -from flask import g -from flask import request -from flask import abort -from flask.ext.principal import identity_changed -from flask.ext.principal import Identity -from flask.ext.principal import AnonymousIdentity - -from models.account import User -from models.account import UserCache - - -def auth_with_key(func): - @wraps(func) - def wrapper(*args, **kwargs): - ip = request.remote_addr - if request.data: - request_args = dict() - _args = request.data.split("&") - for arg in _args: - if arg: - request_args[arg.split("=")[0]] = \ - urllib.unquote(arg.split("=")[1]) - else: - request_args = request.values - - key = request_args.get('_key') - secret = request_args.get('_secret') - if not key and not secret and \ - ip.strip() in current_app.config.get("WHITE_LIST"): - ip = ip.strip() - user = UserCache.get(ip) - if user: - identity_changed.send(current_app._get_current_object(), - identity=Identity(user.uid)) - return func(*args, **kwargs) - else: - identity_changed.send(current_app._get_current_object(), - identity=AnonymousIdentity()) - return abort(401, "invalid _key and _secret") - - path = request.path - - keys = sorted(request_args.keys()) - req_args = [request_args[k] for k in keys - if str(k) not in ("_key", "_secret")] - current_app.logger.debug('args is %s' % req_args) - user, authenticated = User.query.authenticate_with_key( - key, secret, req_args, path) - if user and authenticated: - identity_changed.send(current_app._get_current_object(), - identity=Identity(user.get("uid"))) - return func(*args, **kwargs) - else: - identity_changed.send(current_app._get_current_object(), - identity=AnonymousIdentity()) - return abort(401, "invalid _key and _secret") - - return wrapper diff --git a/lib/ci.py b/lib/ci.py deleted file mode 100644 index f69991f..0000000 --- a/lib/ci.py +++ /dev/null @@ -1,704 +0,0 @@ -# -*- coding:utf-8 -*- - - -import uuid -import time -import datetime -import json - -from flask import current_app -from flask import abort -from sqlalchemy import or_ - -from extensions import db -from extensions import rd -from models.ci import CI -from models.ci_relation import CIRelation -from models.ci_type import CITypeAttribute -from models.ci_type import CITypeCache -from models.ci_type import CITypeSpecCache -from models.history import CIAttributeHistory -from models.attribute import CIAttributeCache -from lib.const import TableMap -from lib.const import type_map -from lib.value import AttributeValueManager -from lib.history import CIAttributeHistoryManger -from lib.history import CIRelationHistoryManager -from lib.query_sql import QUERY_HOSTS_NUM_BY_PRODUCT -from lib.query_sql import QUERY_HOSTS_NUM_BY_BU -from lib.query_sql import QUERY_HOSTS_NUM_BY_PROJECT -from lib.query_sql import QUERY_CIS_BY_IDS -from lib.query_sql import QUERY_CIS_BY_VALUE_TABLE -from tasks.cmdb import ci_cache -from tasks.cmdb import ci_delete - - -class CIManager(object): - """ manage CI interface - """ - - def __init__(self): - pass - - def get_ci_by_id(self, ci_id, ret_key="name", - fields=None, need_children=True, use_master=False): - """@params: `ret_key` is one of 'name', 'id', 'alias' - `fields` is list of attribute name/alias/id - """ - ci = CI.query.get(ci_id) or \ - abort(404, "CI {0} is not existed".format(ci_id)) - - res = dict() - - if need_children: - children = self.get_children(ci_id, ret_key=ret_key) # one floor - res.update(children) - ci_type = CITypeCache.get(ci.type_id) - res["ci_type"] = ci_type.type_name - uniq_key = CIAttributeCache.get(ci_type.uniq_id) - if not fields: # fields are all attributes - attr_ids = db.session.query(CITypeAttribute.attr_id).filter_by( - type_id=ci.type_id) - fields = [CIAttributeCache.get(_.attr_id).attr_name - for _ in attr_ids] - - if uniq_key.attr_name not in fields: - fields.append(uniq_key.attr_name) - if fields: - value_manager = AttributeValueManager() - _res = value_manager._get_attr_values( - fields, ci_id, - ret_key=ret_key, uniq_key=uniq_key, use_master=use_master) - res.update(_res) - res['_type'] = ci_type.type_id - res['_id'] = ci_id - return res - - def get_ci_by_ids(self, ci_id_list, ret_key="name", fields=None): - result = list() - for ci_id in ci_id_list: - res = self.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) - result.append(res) - return result - - def get_children(self, ci_id, ret_key='name', relation_type="contain"): - second_cis = db.session.query(CIRelation.second_ci_id).filter( - CIRelation.first_ci_id == ci_id).filter(or_( - CIRelation.relation_type == relation_type, - CIRelation.relation_type == "deploy")) - second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis) - ci_types = {} - for ci_id in second_ci_ids: - type_id = db.session.query(CI.type_id).filter( - CI.ci_id == ci_id).first().type_id - if type_id not in ci_types: - ci_types[type_id] = [ci_id] - else: - ci_types[type_id].append(ci_id) - res = {} - for type_id in ci_types: - ci_type = CITypeCache.get(type_id) - children = get_cis_by_ids(map(str, ci_types.get(type_id)), - ret_key=ret_key) - res[ci_type.type_name] = children - return res - - def get_cis_by_type(self, type_id, ret_key="name", fields="", - page=1, per_page=None): - if per_page is None: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - cis = db.session.query(CI.ci_id).filter(CI.type_id == type_id) - numfound = cis.count() - cis = cis.offset((page - 1) * per_page).limit(per_page) - res = list() - ci_ids = [str(ci.ci_id) for ci in cis] - if ci_ids: - res = get_cis_by_ids(ci_ids, ret_key, fields) - return numfound, page, res - - def ci_is_exist(self, ci_type, unique_key, unique): - table = TableMap(attr_name=unique_key.attr_name).table - unique = db.session.query(table).filter( - table.attr_id == unique_key.attr_id).filter( - table.value == unique).first() - if unique: - return db.session.query(CI).filter( - CI.ci_id == unique.ci_id).first() - - def _delete_ci_by_id(self, ci_id): - db.session.query(CI.ci_id).filter(CI.ci_id == ci_id).delete() - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("delete ci is error, {0}".format(str(e))) - - def add(self, ci_type_name, exist_policy="replace", - _no_attribute_policy="ignore", **ci_dict): - ci_existed = False - ci_type = CITypeCache.get(ci_type_name) or \ - abort(404, "CIType {0} is not existed".format(ci_type_name)) - - unique_key = CIAttributeCache.get(ci_type.uniq_id) \ - or abort(400, 'illegality unique attribute') - - unique = ci_dict.get(unique_key.attr_name) \ - or abort(400, '{0} missing'.format(unique_key.attr_name)) - - old_ci = self.ci_is_exist(ci_type, unique_key, unique) - if old_ci is not None: - ci_existed = True - if exist_policy == 'reject': - return abort(400, 'CI is existed') - if old_ci.type_id != ci_type.type_id: # update ci_type - old_ci.type_id = ci_type.type_id - db.session.add(old_ci) - db.session.flush() - ci = old_ci - else: - if exist_policy == 'need': - return abort(404, 'CI {0} not exist'.format(unique)) - ci = CI() - ci.type_id = ci_type.type_id - _uuid = uuid.uuid4().hex - ci.uuid = _uuid - ci.created_time = datetime.datetime.now() - db.session.add(ci) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error('add CI error: {0}'.format(str(e))) - return abort(400, 'add CI error') - value_manager = AttributeValueManager() - histories = list() - for p, v in ci_dict.items(): - ret, res = value_manager.add_attr_value( - p, v, ci.ci_id, ci_type, - _no_attribute_policy=_no_attribute_policy, - ci_existed=ci_existed) - if not ret: - db.session.rollback() - if not ci_existed: - self.delete(ci.ci_id) - current_app.logger.info(res) - return abort(400, res) - if res is not None: - histories.append(res) - try: - db.session.commit() - except Exception as e: - current_app.logger.error(str(e)) - db.session.rollback() - if not ci_existed: # only add - self.delete(ci.ci_id) - return abort(400, "add CI error") - his_manager = CIAttributeHistoryManger() - his_manager.add(ci.ci_id, histories) - ci_cache.apply_async([ci.ci_id], queue="cmdb_async") - return ci.ci_id - - def update_unique_value(self, ci_id, args): - ci = self.get_ci_by_id(ci_id, need_children=False) - unique_key = ci.get("unique") - attr = CIAttributeCache.get(unique_key) - table_key = "index_{0}".format(attr.value_type) \ - if attr.is_index else attr.value_type - value_table = type_map.get("table").get(table_key) - v = args.get(unique_key) - if value_table and v: - item = db.session.query(value_table).filter( - value_table.ci_id == ci_id).filter( - value_table.attr_id == attr.attr_id).first() - if item: - converter = type_map.get("converter").get(attr.value_type) - try: - item.value = converter(v) - except: - return abort(400, "value is illegal") - db.session.add(item) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error(str(e)) - return abort(400, "update unique failed") - ci_cache.apply_async([ci_id], queue="cmdb_async") - - def delete(self, ci_id): - ci = db.session.query(CI).filter(CI.ci_id == ci_id).first() - if ci is not None: - attrs = db.session.query(CITypeAttribute.attr_id).filter( - CITypeAttribute.type_id == ci.type_id).all() - attr_names = [] - for attr in attrs: - attr_names.append(CIAttributeCache.get(attr.attr_id).attr_name) - attr_names = set(attr_names) - for attr_name in attr_names: - Table = TableMap(attr_name=attr_name).table - db.session.query(Table).filter(Table.ci_id == ci_id).delete() - db.session.query(CIRelation).filter( - CIRelation.first_ci_id == ci_id).delete() - db.session.query(CIRelation).filter( - CIRelation.second_ci_id == ci_id).delete() - # db.session.query(CIAttributeHistory).filter( - # CIAttributeHistory.ci_id == ci_id).delete() - - db.session.flush() - db.session.delete(ci) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("delete CI error, {0}".format(str(e))) - return abort(400, "delete CI error, {0}".format(str(e))) - # todo: write history - ci_delete.apply_async([ci.ci_id], queue="cmdb_async") - return ci_id - return abort(404, "CI {0} not found".format(ci_id)) - - def add_heartbeat(self, ci_type, unique): - ci_type = CITypeCache.get(ci_type) - if not ci_type: - return 'error' - uniq_key = CIAttributeCache.get(ci_type.uniq_id) - Table = TableMap(attr_name=uniq_key.attr_name).table - ci_id = db.session.query(Table.ci_id).filter( - Table.attr_id == uniq_key.attr_id).filter( - Table.value == unique).first() - if ci_id is None: - return 'error' - ci = db.session.query(CI).filter(CI.ci_id == ci_id.ci_id).first() - if ci is None: - return 'error' - - ci.heartbeat = datetime.datetime.now() - - db.session.add(ci) - db.session.commit() - return "ok" - - def get_heartbeat(self, page, type_id, agent_status=None): - query = db.session.query(CI.ci_id, CI.heartbeat) - expire = datetime.datetime.now() - datetime.timedelta(minutes=72) - if type_id: - query = query.filter(CI.type_id == type_id) - else: - query = query.filter(db.or_(CI.type_id == 7, CI.type_id == 8)) - if agent_status == -1: - query = query.filter(CI.heartbeat == None) - elif agent_status == 0: - query = query.filter(CI.heartbeat <= expire) - elif agent_status == 1: - query = query.filter(CI.heartbeat > expire) - numfound = query.count() - per_page_count = current_app.config.get("DEFAULT_PAGE_COUNT") - cis = query.offset((page - 1) * per_page_count).limit( - per_page_count).all() - ci_ids = [ci.ci_id for ci in cis] - heartbeat_dict = {} - for ci in cis: - if agent_status is not None: - heartbeat_dict[ci.ci_id] = agent_status - else: - if ci.heartbeat is None: - heartbeat_dict[ci.ci_id] = -1 - elif ci.heartbeat <= expire: - heartbeat_dict[ci.ci_id] = 0 - else: - heartbeat_dict[ci.ci_id] = 1 - current_app.logger.debug(heartbeat_dict) - ci_ids = map(str, ci_ids) - res = get_cis_by_ids(ci_ids, fields=["hostname", "private_ip"]) - result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"), - heartbeat_dict.get(i.get("_id"))) for i in res - if i.get("private_ip")] - return numfound, result - - -class CIRelationManager(object): - """ - manage relation between CIs - """ - - def __init__(self): - pass - - @property - def relation_types(self): - """ all CIType relation types - """ - from lib.const import CI_RELATION_TYPES - - return CI_RELATION_TYPES - - def get_second_cis(self, first_ci, relation_type="contain", - page=1, per_page=None, **kwargs): - if per_page is None: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - second_cis = db.session.query( - CI.ci_id).join( - CIRelation, CIRelation.second_ci_id == CI.ci_id).filter( - CIRelation.first_ci_id == first_ci).filter( - CIRelation.relation_type == relation_type) - if kwargs: # special for devices - second_cis = self._query_wrap_for_device(second_cis, **kwargs) - numfound = second_cis.count() - second_cis = second_cis.offset( - (page - 1) * per_page).limit(per_page).all() - ci_ids = [str(son.ci_id) for son in second_cis] - total = len(ci_ids) - result = get_cis_by_ids(ci_ids) - return numfound, total, result - - def get_grandsons(self, ci_id, page=1, per_page=None, **kwargs): - if per_page is None: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - children = db.session.query(CIRelation.second_ci_id).filter( - CIRelation.first_ci_id == ci_id).subquery() - grandsons = db.session.query(CIRelation.second_ci_id).join( - children, - children.c.second_ci_id == CIRelation.first_ci_id).subquery() - grandsons = db.session.query(CI.ci_id).join( - grandsons, grandsons.c.second_ci_id == CI.ci_id) - if kwargs: - grandsons = self._query_wrap_for_device(grandsons, **kwargs) - - numfound = grandsons.count() - grandsons = grandsons.offset( - (page - 1) * per_page).limit(per_page).all() - if not grandsons: - return 0, 0, [] - ci_ids = [str(son.ci_id) for son in grandsons] - total = len(ci_ids) - result = get_cis_by_ids(ci_ids) - - return numfound, total, result - - def _sort_handler(self, sort_by, query_sql): - - if sort_by.startswith("+"): - sort_type = "asc" - sort_by = sort_by[1:] - elif sort_by.startswith("-"): - sort_type = "desc" - sort_by = sort_by[1:] - else: - sort_type = "asc" - attr = CIAttributeCache.get(sort_by) - if attr is None: - return query_sql - - attr_id = attr.attr_id - Table = TableMap(attr_name=sort_by).table - - CI_table = query_sql.subquery() - query_sql = db.session.query(CI_table.c.ci_id, Table.value).join( - Table, Table.ci_id == CI_table.c.ci_id).filter( - Table.attr_id == attr_id).order_by( - getattr(Table.value, sort_type)()) - - return query_sql - - def _query_wrap_for_device(self, query_sql, **kwargs): - _type = kwargs.pop("_type", False) or kwargs.pop("type", False) \ - or kwargs.pop("ci_type", False) - if _type: - ci_type = CITypeCache.get(_type) - if ci_type is None: - return - query_sql = query_sql.filter(CI.type_id == ci_type.type_id) - - for k, v in kwargs.iteritems(): - attr = CIAttributeCache.get(k) - if attr is None: - continue - Table = TableMap(attr_name=k).table - CI_table = query_sql.subquery() - query_sql = db.session.query(CI_table.c.ci_id).join( - Table, Table.ci_id == CI_table.c.ci_id).filter( - Table.attr_id == attr.attr_id).filter( - Table.value.ilike(v.replace("*", "%"))) - - current_app.logger.debug(query_sql) - sort_by = kwargs.pop("sort", False) - if sort_by: - query_sql = self._sort_handler(sort_by, query_sql) - return query_sql - - def get_great_grandsons(self, ci_id, page=1, per_page=None, **kwargs): - if per_page is None: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - - children = db.session.query(CIRelation.second_ci_id).filter( - CIRelation.first_ci_id == ci_id).subquery() - grandsons = db.session.query(CIRelation.second_ci_id).join( - children, - children.c.second_ci_id == CIRelation.first_ci_id).subquery() - great_grandsons = db.session.query(CIRelation.second_ci_id).join( - grandsons, - grandsons.c.second_ci_id == CIRelation.first_ci_id).subquery() - great_grandsons = db.session.query(CI.ci_id).join( - great_grandsons, great_grandsons.c.second_ci_id == CI.ci_id) - if kwargs: - great_grandsons = self._query_wrap_for_device( - great_grandsons, **kwargs) - if great_grandsons is None: - return 0, 0, [] - numfound = great_grandsons.count() - great_grandsons = great_grandsons.offset( - (page - 1) * per_page).limit(per_page).all() - ci_ids = [str(son.ci_id) for son in great_grandsons] - total = len(ci_ids) - result = get_cis_by_ids(ci_ids) - - return numfound, total, result - - def get_first_cis(self, second_ci, relation_type="contain", - page=1, per_page=None): - """only for CI Type - """ - if per_page is None: - per_page = current_app.config.get("DEFAULT_PAGE_COUNT") - first_cis = db.session.query(CIRelation.first_ci_id).filter( - CIRelation.second_ci_id == second_ci).filter( - CIRelation.relation_type == relation_type) - numfound = first_cis.count() - first_cis = first_cis.offset( - (page - 1) * per_page).limit(per_page).all() - result = [] - first_ci_ids = [str(first_ci.first_ci_id) for first_ci in first_cis] - total = len(first_ci_ids) - if first_ci_ids: - result = get_cis_by_ids(first_ci_ids) - return numfound, total, result - - def get_grandfather(self, ci_id, relation_type="contain"): - """only for CI Type - """ - grandfather = db.session.query(CIRelation.first_ci_id).filter( - CIRelation.second_ci_id.in_(db.session.query( - CIRelation.first_ci_id).filter( - CIRelation.second_ci_id == ci_id).filter( - CIRelation.relation_type == relation_type))).filter( - CIRelation.relation_type == relation_type).first() - if grandfather: - return CIManager().get_ci_by_id(grandfather.first_ci_id, - need_children=False) - - def add(self, first_ci, second_ci, more=None, relation_type="contain"): - ci = db.session.query(CI.ci_id).filter(CI.ci_id == first_ci).first() - if ci is None: - return abort(404, "first_ci {0} is not existed".format(first_ci)) - c = db.session.query(CI.ci_id).filter(CI.ci_id == second_ci).first() - if c is None: - return abort(404, "second_ci {0} is not existed".format( - second_ci)) - existed = db.session.query(CIRelation.cr_id).filter( - CIRelation.first_ci_id == first_ci).filter( - CIRelation.second_ci_id == second_ci).first() - if existed is not None: - return existed.cr_id - cr = CIRelation() - cr.first_ci_id = first_ci - cr.second_ci_id = second_ci - if more is not None: - cr.more = more - cr.relation_type = relation_type - db.session.add(cr) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("add CIRelation is error, {0}".format( - str(e))) - return abort(400, "add CIRelation is error, {0}".format(str(e))) - # write history - his_manager = CIRelationHistoryManager() - his_manager.add(cr.cr_id, cr.first_ci_id, cr.second_ci_id, - relation_type, operate_type="add") - return cr.cr_id - - def delete(self, cr_id): - cr = db.session.query(CIRelation).filter( - CIRelation.cr_id == cr_id).first() - cr_id = cr.cr_id - first_ci = cr.first_ci_id - second_ci = cr.second_ci_id - if cr is not None: - db.session.delete(cr) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "delete CIRelation is error, {0}".format(str(e))) - return abort( - 400, "delete CIRelation is error, {0}".format(str(e))) - his_manager = CIRelationHistoryManager() - his_manager.add(cr_id, first_ci, second_ci, - cr.relation_type, operate_type="delete") - return True - return abort(404, "CI relation is not existed") - - def delete_2(self, first_ci, second_ci): - cr = db.session.query(CIRelation).filter( - CIRelation.first_ci_id == first_ci).filter( - CIRelation.second_ci_id == second_ci).first() - return self.delete(cr.cr_id) - - -class HostNumStatis(object): - def __init__(self): - pass - - def get_hosts_by_project(self, project_id_list=None): - res = {} - if not project_id_list: - project = CITypeCache.get("project") - projects = db.session.query(CI.ci_id).filter( - CI.type_id == project.type_id).all() - project_id_list = (project.ci_id for project in projects) - project_id_list = map(str, project_id_list) - project_ids = ",".join(project_id_list) - nums = db.session.execute(QUERY_HOSTS_NUM_BY_PROJECT.format( - "".join(["(", project_ids, ")"]))).fetchall() - if nums: - for ci_id in project_id_list: - res[int(ci_id)] = 0 - for ci_id, num in nums: - res[ci_id] = num - return res - - def get_hosts_by_product(self, product_id_list=None): - res = {} - if not product_id_list: - product = CITypeCache.get("product") - products = db.session.query(CI.ci_id).filter( - CI.type_id == product.type_id).all() - product_id_list = (product.ci_id for product in products) - product_id_list = map(str, product_id_list) - product_ids = ",".join(product_id_list) - nums = db.session.execute(QUERY_HOSTS_NUM_BY_PRODUCT.format( - "".join(["(", product_ids, ")"]))).fetchall() - if nums: - for ci_id in product_id_list: - res[int(ci_id)] = 0 - for ci_id, num in nums: - res[ci_id] = num - return res - - def get_hosts_by_bu(self, bu_id_list=None): - res = {} - if not bu_id_list: - bu = CITypeCache.get("bu") - bus = db.session.query(CI.ci_id).filter( - CI.type_id == bu.type_id).all() - bu_id_list = (bu.ci_id for bu in bus) - bu_id_list = map(str, bu_id_list) - bu_ids = ",".join(bu_id_list) - current_app.logger.debug(QUERY_HOSTS_NUM_BY_BU.format( - "".join(["(", bu_ids, ")"]))) - if not bu_ids: - return res - nums = db.session.execute( - QUERY_HOSTS_NUM_BY_BU.format( - "".join(["(", bu_ids, ")"]))).fetchall() - if nums: - for ci_id in bu_id_list: - res[int(ci_id)] = 0 - for ci_id, num in nums: - res[ci_id] = num - return res - - -def get_cis_by_ids(ci_ids, ret_key="name", fields="", value_tables=None): - """ argument ci_ids are string list of CI instance ID, eg. ['1', '2'] - """ - if not ci_ids: - return [] - start = time.time() - ci_id_tuple = tuple(map(int, ci_ids)) - res = rd.get(ci_id_tuple) - if res is not None and None not in res and ret_key == "name": - res = map(json.loads, res) - if not fields: - return res - else: - _res = [] - for d in res: - _d = dict() - _d["_id"], _d["_type"] = d.get("_id"), d.get("_type") - _d["ci_type"] = d.get("ci_type") - for field in fields: - _d[field] = d.get(field) - _res.append(_d) - current_app.logger.debug("filter time: %s" % (time.time() - start)) - return _res - current_app.logger.warning("cache not hit...............") - if not fields: - _fields = "" - else: - _fields = list() - for field in fields: - attr = CIAttributeCache.get(field) - if attr is not None: - _fields.append(str(attr.attr_id)) - _fields = "WHERE A.attr_id in ({0})".format(",".join(_fields)) - ci_ids = ",".join(ci_ids) - if value_tables is None: - value_tables = type_map["table_name"].values() - current_app.logger.debug(value_tables) - value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, - ci_ids) - for value_table in value_tables]) - query_sql = QUERY_CIS_BY_IDS.format(ci_ids, _fields, value_sql) - current_app.logger.debug(query_sql) - start = time.time() - hosts = db.session.execute(query_sql).fetchall() - current_app.logger.info("get cis time is: {0}".format( - time.time() - start)) - - ci_list = set() - res = list() - ci_dict = dict() - start = time.time() - for ci_id, type_id, attr_id, attr_name, \ - attr_alias, value, value_type, is_multivalue in hosts: - if ci_id not in ci_list: - ci_dict = dict() - ci_type = CITypeSpecCache.get(type_id) - ci_dict["_id"] = ci_id - ci_dict["_type"] = type_id - ci_dict["ci_type"] = ci_type.type_name - ci_dict["ci_type_alias"] = ci_type.type_alias - ci_list.add(ci_id) - res.append(ci_dict) - if ret_key == "name": - if is_multivalue: - if isinstance(ci_dict.get(attr_name), list): - ci_dict[attr_name].append(value) - else: - ci_dict[attr_name] = [value] - else: - ci_dict[attr_name] = value - elif ret_key == "alias": - if is_multivalue: - if isinstance(ci_dict.get(attr_alias), list): - ci_dict[attr_alias].append(value) - else: - ci_dict[attr_alias] = [value] - else: - ci_dict[attr_alias] = value - elif ret_key == "id": - if is_multivalue: - if isinstance(ci_dict.get(attr_id), list): - ci_dict[attr_id].append(value) - else: - ci_dict[attr_id] = [value] - else: - ci_dict[attr_id] = value - - current_app.logger.debug("result parser time is: {0}".format( - time.time() - start)) - return res \ No newline at end of file diff --git a/lib/ci_type.py b/lib/ci_type.py deleted file mode 100644 index 51f70ca..0000000 --- a/lib/ci_type.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask import current_app -from flask import abort - -from extensions import db -from models import row2dict -from models.ci_type import CITypeAttribute -from models.ci_type import CIType -from models.ci_type import CITypeAttributeCache -from models.ci_type import CITypeCache -from models.ci_type_relation import CITypeRelation -from models.attribute import CIAttributeCache -from lib.attribute import AttributeManager - - -class CITypeAttributeManager(object): - """ - manage CIType's attributes, include query, add, update, delete - """ - - def __init__(self): - pass - - def get_attributes_by_type_id(self, type_id): - attrs = CITypeAttributeCache.get(type_id) - attr_manager = AttributeManager() - result = list() - for attr in attrs: - attr_dict = attr_manager.get_attribute_by_id(attr.attr_id) - attr_dict["is_required"] = attr.is_required - attr_dict["order"] = attr.order - result.append(attr_dict) - return result - - def add(self, type_id, attr_ids=None, is_required=False): - """ - add attributes to CIType, attr_ids are list - """ - if not attr_ids or not isinstance(attr_ids, list): - return abort(500, "attr_ids must be required") - ci_type = CITypeCache.get(type_id) - if ci_type is None: - return abort(404, "CIType ID({0}) is not existed".format(type_id)) - for attr_id in attr_ids: - attr = CIAttributeCache.get(attr_id) - if attr is None: - return abort(404, - "attribute id {0} is not existed".format(attr_id)) - existed = db.session.query(CITypeAttribute.attr_id).filter_by( - type_id=type_id).filter_by(attr_id=attr_id).first() - if existed is not None: - continue - current_app.logger.debug(attr_id) - db.session.add(CITypeAttribute( - type_id=type_id, attr_id=attr_id, is_required=is_required)) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "add attribute to CIType is error, {0}".format(str(e))) - return abort( - 500, "add attribute to CIType is error, maybe duplicate entry") - - CITypeAttributeCache.clean(type_id) - return True - - def delete(self, type_id, attr_ids=None): - """ - delete attributes at CIType, attr_ids are list - """ - if not attr_ids or not isinstance(attr_ids, list): - return abort( - 500, "delete attribute of CIType, attr_ids must be required") - ci_type = CITypeCache.get(type_id) - if ci_type is None: - return abort( - 404, "CIType ID({0}) is not existed".format(type_id)) - for attr_id in attr_ids: - attr = CIAttributeCache.get(attr_id) - if attr is None: - return abort( - 404, "attribute id {0} is not existed".format(attr_id)) - db.session.query(CITypeAttribute).filter_by( - type_id=type_id).filter_by(attr_id=attr_id).delete() - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "delete attributes of CIType is error, {0}".format(str(e))) - return abort(500, "delete attributes of CIType is error") - CITypeAttributeCache.clean(type_id) - return True - - -class CITypeManager(object): - """ - manage CIType - """ - - def __init__(self): - pass - - def get_citypes(self, type_name=None): - ci_types = db.session.query(CIType).all() if type_name is None else \ - db.session.query(CIType).filter( - CIType.type_name.ilike("%{0}%".format(type_name))).all() - res = list() - for ci_type in ci_types: - type_dict = row2dict(ci_type) - type_dict["uniq_key"] = CIAttributeCache.get( - type_dict["uniq_id"]).attr_name - res.append(type_dict) - return res - - def query(self, _type): - citype = CITypeCache.get(_type) - if citype: - return row2dict(citype) - return abort(404, "citype is not found") - - def add(self, type_name, type_alias, _id=None, unique=None, - icon_url="", enabled=True): - uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique) - if uniq_key is None: - return False, "uniq_key is not existed" - citype = CITypeCache.get(type_name) - if citype: - return False, "this CIType {0} is existed".format(type_name) - _citype = CIType() - _citype.type_name = type_name - _citype.type_alias = type_alias - _citype.uniq_id = uniq_key.attr_id - _citype.enabled = enabled - _citype.icon_url = icon_url - db.session.add(_citype) - db.session.flush() - _citype_attr = CITypeAttribute() - _citype_attr.attr_id = uniq_key.attr_id - _citype_attr.type_id = _citype.type_id - _citype_attr.is_required = True - db.session.add(_citype_attr) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("add CIType is error, {0}".format(str(e))) - return False, str(e) - CITypeCache.clean(type_name) - return True, _citype.type_id - - def update(self, type_id, type_name, type_alias, _id=None, unique=None, - icon_url="", enabled=None): - citype = CITypeCache.get(type_id) - if citype is None: - return False, "CIType {0} is not existed".format(type_name) - uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique) - if uniq_key is not None: - citype.uniq_id = uniq_key.attr_id - citype_attr = db.session.query(CITypeAttribute).filter( - CITypeAttribute.type_id == type_id).filter( - CITypeAttribute.attr_id == uniq_key.attr_id).first() - if citype_attr is None: - citype_attr = CITypeAttribute() - citype_attr.attr_id = uniq_key.attr_id - citype_attr.type_id = type_id - citype_attr.is_required = True - db.session.add(citype_attr) - if type_name: - citype.type_name = type_name - if type_alias: - citype.type_alias = type_alias - if icon_url: - citype.icon_url = icon_url - if enabled is not None: - citype.enabled = enabled - db.session.add(citype) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error("add CIType is error, {0}".format(str(e))) - return False, str(e) - CITypeCache.clean(type_id) - return True, type_id - - def set_enabled(self, type_id, enabled=True): - citype = CITypeCache.get(type_id) - if citype is None: - return abort(404, "CIType[{0}] is not existed".format(type_id)) - citype.enabled = enabled - db.session.add(citype) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "set CIType enabled is error, {0}".format(str(e))) - return abort(500, str(e)) - return type_id - - def delete(self, type_id): - citype = db.session.query(CIType).filter_by(type_id=type_id).first() - type_name = citype.type_name - if citype: - db.session.delete(citype) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "delete CIType is error, {0}".format(str(e))) - return abort(500, str(e)) - CITypeCache.clean(type_id) - return "CIType {0} deleted".format(type_name) - return abort(404, "CIType is not existed") - - -class CITypeRelationManager(object): - """ - manage relation between CITypes - """ - - def __init__(self): - pass - - @property - def relation_types(self): - """ all CIType relation types - """ - from lib.const import CITYPE_RELATION_TYPES - - return CITYPE_RELATION_TYPES - - def get_children(self, parent_id): - children = db.session.query(CITypeRelation).filter( - CITypeRelation.parent_id == parent_id).all() - result = [] - for child in children: - ctr_id = child.ctr_id - citype = CITypeCache.get(child.child_id) - citype_dict = row2dict(citype) - citype_dict["ctr_id"] = ctr_id - manager = CITypeAttributeManager() - citype_dict["attributes"] = manager.get_attributes_by_type_id( - citype.type_id) - citype_dict["relation_type"] = child.relation_type - result.append(citype_dict) - return result - - def get_parents(self, child_id): - parents = db.session.query(CITypeRelation).filter( - CITypeRelation.child_id == child_id).all() - result = [] - for parent in parents: - ctr_id = parent.ctr_id - citype = CITypeCache.get(parent.parent_id) - citype_dict = row2dict(citype) - citype_dict["ctr_id"] = ctr_id - manager = CITypeAttributeManager() - citype_dict["attributes"] = manager.get_attributes_by_type_id( - citype.type_id) - citype_dict["relation_type"] = parent.relation_type - result.append(citype_dict) - return result - - def add(self, parent, child, relation_type="contain"): - p = CITypeCache.get(parent) - if p is None: - return abort(404, "parent {0} is not existed".format(parent)) - c = CITypeCache.get(child) - if c is None: - return abort(404, "child {0} is not existed".format(child)) - existed = db.session.query(CITypeRelation.ctr_id).filter_by( - parent_id=parent).filter_by(child_id=child).first() - if existed is not None: - return True, existed.ctr_id - ctr = CITypeRelation() - ctr.parent_id = parent - ctr.child_id = child - ctr.relation_type = relation_type - db.session.add(ctr) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "add CITypeRelation is error, {0}".format(str(e))) - return abort( - 500, "add CITypeRelation is error, {0}".format(str(e))) - return ctr.ctr_id - - def delete(self, ctr_id): - ctr = db.session.query(CITypeRelation).filter( - CITypeRelation.ctr_id == ctr_id).first() - if ctr: - db.session.delete(ctr) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "delete CITypeRelation is error, {0}".format(str(e))) - return abort( - 500, "delete CITypeRelation is error, {0}".format(str(e))) - return True - return abort(404, "CIType relation is not existed") - - def delete_2(self, parent, child): - ctr = db.session.query(CITypeRelation).filter( - CITypeRelation.parent_id == parent).filter( - CITypeRelation.child_id == child).first() - return self.delete(ctr.ctr_id) \ No newline at end of file diff --git a/lib/const.py b/lib/const.py deleted file mode 100644 index ee25660..0000000 --- a/lib/const.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from markupsafe import escape - -from models.attribute import TextChoice -from models.attribute import FloatChoice -from models.attribute import IntegerChoice -from models.attribute import CIAttributeCache -from models.ci_value import CIValueText -from models.ci_value import CIValueInteger -from models.ci_value import CIValueFloat -from models.ci_value import CIValueDateTime -from models.ci_value import CIIndexValueDateTime -from models.ci_value import CIIndexValueFloat -from models.ci_value import CIIndexValueInteger -from models.ci_value import CIIndexValueText - - -def string2int(x): - return int(float(x)) - - -def str2datetime(x): - try: - v = datetime.datetime.strptime(x, "%Y-%m-%d") - return v - except ValueError: - pass - try: - v = datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") - return v - except ValueError: - pass - raise ValueError - - -type_map = { - 'converter': { - 'int': string2int, - 'float': float, - 'text': escape, - 'datetime': str2datetime, - }, - 'choice': { - 'int': IntegerChoice, - 'float': FloatChoice, - 'text': TextChoice, - }, - 'table': { - 'int': CIValueInteger, - 'text': CIValueText, - 'datetime': CIValueDateTime, - 'float': CIValueFloat, - 'index_int': CIIndexValueInteger, - 'index_text': CIIndexValueText, - 'index_datetime': CIIndexValueDateTime, - 'index_float': CIIndexValueFloat, - }, - 'table_name': { - 'int': 'integers', - 'text': 'texts', - 'datetime': 'datetime', - 'float': 'floats', - 'index_int': 'index_integers', - 'index_text': 'index_texts', - 'index_datetime': 'index_datetime', - 'index_float': 'index_floats', - } -} - - -class TableMap(): - def __init__(self, attr_name=None): - self.attr_name = attr_name - - @property - def table(self): - if self.attr_name is not None: - attr = CIAttributeCache.get(self.attr_name) - if attr.is_index: - i = "index_{0}".format(attr.value_type) - else: - i = attr.value_type - return type_map["table"].get(i) - - @property - def table_name(self): - if self.attr_name is not None: - attr = CIAttributeCache.get(self.attr_name) - if attr.is_index: - i = "index_{0}".format(attr.value_type) - else: - i = attr.value_type - return type_map["table_name"].get(i) - - -CITYPE_RELATION_TYPES = ["connect", "deploy", "install", "contain"] -CI_RELATION_TYPES = ["connect", "deploy", "install", "contain"] \ No newline at end of file diff --git a/lib/decorator.py b/lib/decorator.py deleted file mode 100644 index 8cafba7..0000000 --- a/lib/decorator.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding:utf-8 -*- - - -import time -from functools import wraps - -from flask import request -from flask import render_template -from flask import current_app - -from lib.exception import InvalidUsageError - - -def templated(template=None): - def decorator(f): - @wraps(f) - def decorated_function(*args, **kwargs): - template_name = template - if template_name is None: - template_name = request.endpoint.replace('.', '/') + '.html' - ctx = f(*args, **kwargs) - if ctx is None: - ctx = {} - elif not isinstance(ctx, dict): - return ctx - return render_template(template_name, **ctx) - - return decorated_function - - return decorator - - -def argument_required1(*args_required): - from manage import InvalidUsageError - - def decorator(f): - @wraps(f) - def decorated_function(*args, **kwargs): - for arg in args_required: - if request.values.get(arg, None) is None: - raise InvalidUsageError( - "argument {0} is required".format(arg), 400) - return f(*args, **kwargs) - - return decorated_function - - return decorator - - -class argument_required(object): - def __init__(self, *args): - self.args = args - - def __enter__(self): - for arg in self.args: - if not request.values.get(arg): - raise InvalidUsageError( - "argument {0} is required".format(arg), status_code=400) - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - -def url_statistic(f): - @wraps(f) - def decorated_func(*args, **kwargs): - start = time.time() - r = f(*args, **kwargs) - spend = time.time() - start - url = request.path - current_app.logger.info(url) - current_app.logger.info(spend) - return r - return decorated_func \ No newline at end of file diff --git a/lib/exception.py b/lib/exception.py deleted file mode 100644 index a11af85..0000000 --- a/lib/exception.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding:utf-8 -*- - - -class InvalidUsageError(Exception): - status_code = 400 - - def __init__(self, message, status_code=None, payload=None): - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload - - def to_dict(self): - rv = dict(self.payload or ()) - rv['message'] = self.message - return rv \ No newline at end of file diff --git a/lib/history.py b/lib/history.py deleted file mode 100644 index ebcef62..0000000 --- a/lib/history.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from flask import current_app -from flask import g - -from extensions import db -from models.history import OperationRecord -from models.history import CIAttributeHistory -from models.history import CIRelationHistory - - -class CIAttributeHistoryManger(object): - def __init__(self): - pass - - def add(self, ci_id, history_list): - if history_list: - record = OperationRecord() - record.uid = g.user.uid - record.timestamp = datetime.datetime.now() - db.session.add(record) - db.session.commit() - for attr_id, operate_type, old, new in history_list: - history = CIAttributeHistory() - history.attr_id = attr_id - history.operate_type = operate_type - history.old = old - history.new = new - history.ci_id = ci_id - history.record_id = record.record_id - db.session.add(history) - - try: - db.session.commit() - except Exception as e: - db.session.rollback() - db.session.rollback() - current_app.logger.error( - "add attribute history error, {0}".format(str(e))) - return False, "add attribute history error, {0}".format(str(e)) - return True, None - - -class CIRelationHistoryManager(object): - def __init__(self): - pass - - def add(self, relation, first_ci, second_ci, - relation_type, operate_type="add"): - record = OperationRecord() - record.uid = g.user.uid - record.timestamp = datetime.datetime.now() - db.session.add(record) - db.session.flush() - - history = CIRelationHistory() - history.relation = relation - history.record_id = record.record_id - history.operate_type = operate_type - history.first_ci_id = first_ci - history.second_ci_id = second_ci - history.relation_type = relation_type - db.session.add(history) - - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "add relation history is error, {0}".format(str(e))) - return False, "add relation history is error, {0}".format(str(e)) - return True, None diff --git a/lib/query_sql.py b/lib/query_sql.py deleted file mode 100644 index 66e89b1..0000000 --- a/lib/query_sql.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding:utf-8 -*- - - -QUERY_HOSTS_BY_APP = """ - SELECT * - FROM cis - INNER JOIN ci_relations AS cr ON cis.`ci_id`=cr.`second_ci` - WHERE cr.`first_ci` = {0:d} LIMIT {1:d}, {2:d}; -""" - -QUERY_HOSTS_NUM_BY_PROJECT = """ - SELECT cr.first_ci_id, - count(DISTINCT cr.second_ci_id) - FROM ci_relations AS cr - WHERE cr.first_ci_id IN {0} - GROUP BY cr.first_ci_id -""" - -QUERY_HOSTS_NUM_BY_BU = """ - SELECT B.first_ci_id, - count(DISTINCT cr.second_ci_id) - FROM - (SELECT A.first_ci_id, - cr.second_ci_id - FROM - (SELECT cr.first_ci_id, - cis.ci_id - FROM cis - INNER JOIN ci_relations AS cr ON cis.ci_id=cr.second_ci_id - WHERE cr.first_ci_id IN {0}) AS A - INNER JOIN ci_relations AS cr ON cr.first_ci_id=A.ci_id) AS B - INNER JOIN ci_relations AS cr ON B.second_ci_id=cr.first_ci_id - GROUP BY B.first_ci_id -""" - -QUERY_HOSTS_NUM_BY_PRODUCT = """ - SELECT A.first_ci_id, - count(DISTINCT cr.second_ci_id) - FROM - (SELECT cr.first_ci_id, - cis.ci_id - FROM cis - INNER JOIN ci_relations AS cr ON cis.ci_id=cr.second_ci_id - WHERE cr.first_ci_id IN {0}) AS A - INNER JOIN ci_relations AS cr ON cr.first_ci_id=A.ci_id - GROUP BY A.first_ci_id; -""" - -QUERY_CIS_BY_VALUE_TABLE = """ - SELECT attr.attr_name, - attr.attr_alias, - attr.value_type, - attr.is_multivalue, - cis.type_id, - {0}.ci_id, - {0}.attr_id, - {0}.value - FROM {0} - INNER JOIN cis ON {0}.ci_id=cis.ci_id - AND {0}.`ci_id` IN ({1}) - INNER JOIN ci_attributes as attr ON attr.attr_id = {0}.attr_id -""" - -QUERY_CIS_BY_IDS = """ - SELECT A.ci_id, - A.type_id, - A.attr_id, - A.attr_name, - A.attr_alias, - A.value, - A.value_type, - A.is_multivalue - FROM - ({2}) AS A {1} - ORDER BY A.ci_id; -""" - -FACET_QUERY1 = """ - SELECT {0}.value, - count({0}.ci_id) - FROM {0} - INNER JOIN ci_attributes AS attr ON attr.attr_id={0}.attr_id - WHERE attr.attr_name="{1}" - GROUP BY {0}.ci_id; -""" - -FACET_QUERY = """ - SELECT {0}.value, - count({0}.ci_id) - FROM {0} - INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id - WHERE {0}.attr_id={2:d} - GROUP BY {0}.value -""" - -QUERY_CI_BY_ATTR_NAME = """ - SELECT {0}.ci_id - FROM {0} - WHERE {0}.attr_id={1:d} - AND {0}.value {2} -""" - -QUERY_CI_BY_TYPE = """ - SELECT cis.ci_id - FROM cis - WHERE cis.type_id in ({0}) -""" \ No newline at end of file diff --git a/lib/search.py b/lib/search.py deleted file mode 100644 index fc9c997..0000000 --- a/lib/search.py +++ /dev/null @@ -1,365 +0,0 @@ -# -*- coding:utf-8 -*- - - -import time - -from flask import current_app - -from lib.const import TableMap -from models.attribute import CIAttributeCache -from models.ci_type import CITypeCache -from extensions import db -from models.ci import CI -from lib.ci import get_cis_by_ids -from lib.query_sql import FACET_QUERY -from lib.query_sql import QUERY_CI_BY_TYPE -from lib.query_sql import QUERY_CI_BY_ATTR_NAME - - -class SearchError(Exception): - def __init__(self, v): - self.v = v - - def __str__(self): - return self.v - - -class Search(object): - def __init__(self, query=None, fl=None, facet_field=None, - page=1, ret_key="name", count=1, sort=None): - self.orig_query = query - self.fl = fl - self.facet_field = facet_field - self.page = page - self.ret_key = ret_key - try: - self.count = int(count) - except ValueError: - self.count = current_app.config.get("DEFAULT_PAGE_COUNT") - self.sort = sort - self.query_sql = "" - self.type_id_list = [] - - def tor_proc(self, key): - tor = list() - if key.startswith("+"): - tor.append('&') - key = key[1:].strip() - elif key.startswith("-"): - tor.append('|') - key = key[1:].strip() - elif key.startswith("~"): - tor.append('~') - key = key[1:].strip() - if not tor: - tor = ['&', ''] - if len(tor) < 2: - tor.append('') - return tor, key - - def attr_name_proc(self, key): - tor, key = self.tor_proc(key) - if key in ('ci_type', 'type', '_type'): - return '_type', 'text', tor, None - if key in ('id', 'ci_id', '_id'): - return '_id', 'text', tor, None - attr = CIAttributeCache.get(key) - if attr is not None: - # if not attr.is_index: - # raise SearchError("{0} is not indexed".format(attr.attr_name)) - field_name = attr.attr_name - return field_name, attr.value_type, tor, attr - else: - raise SearchError("{0} is not existed".format(key)) - - def type_query_handler(self, v, only_type_query): - new_v = [v] - if v.startswith("(") and v.endswith(")"): - new_v = v[1:-1].split(";") - for _v in new_v: - ci_type = CITypeCache.get(_v) - if ci_type is not None: - self.type_id_list.append(str(ci_type.type_id)) - if self.type_id_list: - type_ids = ",".join(self.type_id_list) - _query_sql = QUERY_CI_BY_TYPE.format(type_ids) - if only_type_query: - return _query_sql - else: - return "" - return "" - - def in_query_handler(self, attr, v): - new_v = v[1:-1].split(";") - table_name = TableMap(attr_name=attr.attr_name).table_name - _query_sql = QUERY_CI_BY_ATTR_NAME.format( - table_name, attr.attr_id, - " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format( - _v.replace("*", "%")) for _v in new_v])) - return _query_sql - - def range_query_handler(self, attr, v): - start, end = [x.strip() for x in v[1:-1].split("_TO_")] - table_name = TableMap(attr_name=attr.attr_name).table_name - _query_sql = QUERY_CI_BY_ATTR_NAME.format( - table_name, attr.attr_id, "BETWEEN '{0}' AND '{1}'".format( - start.replace("*", "%"), end.replace("*", "%"))) - return _query_sql - - def comparison_query_handler(self, attr, v): - table_name = TableMap(attr_name=attr.attr_name).table_name - if (v.startswith("<") and not v.startswith("<=")) or \ - (v.startswith(">") and not v.startswith(">=")): - _query_sql = QUERY_CI_BY_ATTR_NAME.format( - table_name, attr.attr_id, "{0} '{1}'".format( - v[0], v[1:].replace("*", "%"))) - elif v.startswith(">=") or v.startswith("<="): - _query_sql = QUERY_CI_BY_ATTR_NAME.format( - table_name, attr.attr_id, "{0} '{1}'".format( - v[:2], v[2:].replace("*", "%"))) - return _query_sql - - def sort_query_handler(self, field, query_sql, only_type_query): - if field is None: - field = "" - if field.startswith("+"): - field = field[1:] - sort_type = "ASC" - elif field.startswith("-"): - field = field[1:] - sort_type = "DESC" - else: - sort_type = "ASC" - - if field in ("_id", "ci_id") or not field: - if only_type_query: - return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id - FROM ({0}) AS B {1}""".format( - query_sql, - "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( - (self.page - 1) * self.count, sort_type, self.count)) - elif self.type_id_list: - self.query_sql = """SELECT B.ci_id - FROM ({0}) AS B {1}""".format( - query_sql, - "INNER JOIN cis on cis.ci_id=B.ci_id " - "WHERE cis.type_id in ({0}) ".format( - ",".join(self.type_id_list))) - return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id - FROM ({0}) AS B {1}""".format( - query_sql, - "INNER JOIN cis on cis.ci_id=B.ci_id " - "WHERE cis.type_id in ({3}) " - "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( - (self.page - 1) * self.count, sort_type, self.count, - ",".join(self.type_id_list))) - else: - self.query_sql = """SELECT B.ci_id - FROM ({0}) AS B {1}""".format( - query_sql, - "INNER JOIN cis on cis.ci_id=B.ci_id ") - return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id - FROM ({0}) AS B {1}""".format( - query_sql, - "INNER JOIN cis on cis.ci_id=B.ci_id " - "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( - (self.page - 1) * self.count, sort_type, self.count)) - else: - attr = CIAttributeCache.get(field) - attr_id = attr.attr_id - - table_name = TableMap(attr_name=attr.attr_name).table_name - _v_query_sql = """SELECT {0}.ci_id, {1}.value FROM - ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id - WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, - query_sql, attr_id) - new_table = _v_query_sql - if only_type_query: - return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \ - "FROM ({0}) AS C " \ - "ORDER BY C.value {2} " \ - "LIMIT {1:d}, {3};".format(new_table, - (self.page - 1) * self.count, - sort_type, self.count) - elif self.type_id_list: - self.query_sql = """SELECT C.ci_id - FROM ({0}) AS C - INNER JOIN cis on cis.ci_id=C.ci_id - WHERE cis.type_id in ({1})""".format( - new_table, - ",".join(self.type_id_list)) - return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id - FROM ({0}) AS C - INNER JOIN cis on cis.ci_id=C.ci_id - WHERE cis.type_id in ({4}) - ORDER BY C.value {2} - LIMIT {1:d}, {3};""".format(new_table, - (self.page - 1) * self.count, - sort_type, self.count, - ",".join(self.type_id_list)) - else: - return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id - FROM ({0}) AS C - ORDER BY C.value {2} - LIMIT {1:d}, {3};""".format(new_table, - (self.page - 1) * self.count, - sort_type, self.count) - - def _wrap_sql(self, tor, alias, _query_sql, query_sql): - if tor[0] == "&": - query_sql = """SELECT * FROM ({0}) as {1} - INNER JOIN ({2}) as {3} USING(ci_id)""".format( - query_sql, alias, _query_sql, alias + "A") - elif tor[0] == "|": - query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format( - query_sql, alias, _query_sql) - elif tor[0] == "~": - query_sql = "SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} " \ - "USING(ci_id) WHERE {3}.ci_id is NULL".format( - query_sql, alias, _query_sql, alias + "A") - return query_sql - - def _execute_sql(self, query_sql, only_type_query): - v_query_sql = self.sort_query_handler(self.sort, query_sql, - only_type_query) - start = time.time() - execute = db.session.execute - current_app.logger.debug(v_query_sql) - res = execute(v_query_sql).fetchall() - end_time = time.time() - current_app.logger.debug("query ci ids time is: {0}".format( - end_time - start)) - numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0] - current_app.logger.debug("statistics ci ids time is: {0}".format( - time.time() - end_time) - ) - return numfound, res - - def query_build_raw(self): - query_sql, alias, tor = "", "A", ["&"] - is_first = True - only_type_query = False - queries = self.orig_query.split(",") - queries = filter(lambda x: x != "", queries) - for q in queries: - if q.startswith("_type"): - queries.remove(q) - queries.insert(0, q) - if len(queries) == 1 or queries[1].startswith("-") or \ - queries[1].startswith("~"): - only_type_query = True - break - current_app.logger.debug(queries) - special = True - for q in queries: - _query_sql = "" - if ":" in q: - k = q.split(":")[0].strip() - v = ":".join(q.split(":")[1:]).strip() - current_app.logger.info(v) - field, field_type, tor, attr = self.attr_name_proc(k) - if field == "_type": - _query_sql = self.type_query_handler(v, only_type_query) - current_app.logger.debug(_query_sql) - elif field == "_id": # exclude all others - _ci_ids = [str(v)] - ci = db.session.query(CI.ci_id).filter( - CI.ci_id == int(v)).first() - if ci is not None: - return 1, _ci_ids - elif field: - if attr is None: - raise SearchError("{0} is not found".format(field)) - # in query - if v.startswith("(") and v.endswith(")"): - _query_sql = self.in_query_handler(attr, v) - # range query - elif v.startswith("[") and v.endswith("]") and "_TO_" in v: - _query_sql = self.range_query_handler(attr, v) - # comparison query - elif v.startswith(">=") or v.startswith("<=") or \ - v.startswith(">") or v.startswith("<"): - _query_sql = self.comparison_query_handler(attr, v) - else: - table_name = \ - TableMap(attr_name=attr.attr_name).table_name - _query_sql = QUERY_CI_BY_ATTR_NAME.format( - table_name, attr.attr_id, - 'LIKE "{0}"'.format(v.replace("*", "%"))) - else: - return 0, [] - elif q: - return 0, [] - - if is_first and _query_sql and not only_type_query: - query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, - alias) - is_first = False - alias += "A" - elif only_type_query and special: - is_first = False - special = False - query_sql = _query_sql - elif _query_sql: - query_sql = self._wrap_sql(tor, alias, _query_sql, query_sql) - alias += "AA" - - _start = time.time() - if query_sql: - self.query_sql = query_sql - current_app.logger.debug(query_sql) - numfound, res = self._execute_sql(query_sql, only_type_query) - current_app.logger.info("query ci ids is: {0}".format( - time.time() - _start)) - return numfound, [_res[0] for _res in res] - return 0, [] - - def facet_build(self): - facet = {} - for f in self.facet_field: - k, field_type, _, attr = self.attr_name_proc(f) - if k: - table_name = TableMap(attr_name=k).table_name - query_sql = FACET_QUERY.format( - table_name, self.query_sql, attr.attr_id) - current_app.logger.debug(query_sql) - result = db.session.execute(query_sql).fetchall() - facet[k] = result - facet_result = dict() - for k, v in facet.items(): - if not k.startswith('_'): - a = getattr(CIAttributeCache.get(k), "attr_%s" % self.ret_key) - facet_result[a] = list() - for f in v: - if f[1] != 0: - facet_result[a].append((f[0], f[1], a)) - return facet_result - - def fl_build(self): - _fl = list() - for f in self.fl: - k, _, _, _ = self.attr_name_proc(f) - if k: - _fl.append(k) - return _fl - - def search(self): - numfound, ci_ids = self.query_build_raw() - ci_ids = map(str, ci_ids) - _fl = self.fl_build() - - if self.facet_field and numfound: - facet = self.facet_build() - else: - facet = dict() - - response, counter = [], {} - if ci_ids: - response = get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl) - for res in response: - ci_type = res.get("ci_type") - if ci_type not in counter.keys(): - counter[ci_type] = 0 - counter[ci_type] += 1 - total = len(response) - return response, counter, total, self.page, numfound, facet diff --git a/lib/template/__init__.py b/lib/template/__init__.py deleted file mode 100644 index 44d37d3..0000000 --- a/lib/template/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding:utf-8 -*- \ No newline at end of file diff --git a/lib/template/filters.py b/lib/template/filters.py deleted file mode 100644 index 149d4f7..0000000 --- a/lib/template/filters.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding:utf-8 -*- - - -def convert_to_list(v): - if isinstance(v, list): - return v - if isinstance(v, tuple): - return list(v) - return [v, ] diff --git a/lib/value.py b/lib/value.py deleted file mode 100644 index 4c98469..0000000 --- a/lib/value.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from flask import current_app - -from extensions import db -from models.attribute import CIAttributeCache -from lib.attribute import AttributeManager -from lib.const import type_map -from lib.const import TableMap - - -class AttributeValueManager(object): - """ - manage CI attribute values - """ - - def __init__(self): - pass - - def _get_attr(self, key): - """key is one of attr_id, attr_name and attr_alias - """ - attr = CIAttributeCache.get(key) - return attr - - def _get_attr_values(self, fields, ci_id, - ret_key="name", - uniq_key=None, - use_master=False): - res = dict() - for field in fields: - attr = CIAttributeCache.get(field) - if not attr: - current_app.logger.warn('attribute %s not found' % field) - return res - table = TableMap(attr_name=attr.attr_name).table - if use_master: - rs = db.session().using_bind("master").query( - table.value).filter_by(ci_id=ci_id).filter_by( - attr_id=attr.attr_id) - else: - rs = db.session.query(table.value).filter_by( - ci_id=ci_id).filter_by(attr_id=attr.attr_id) - field_name = getattr(attr, "attr_{0}".format(ret_key)) - try: - if attr.is_multivalue: - if attr.value_type == 'datetime': - res[field_name] = [datetime.datetime.strftime( - x.value, '%Y-%m-%d %H:%M:%S') for x in rs.all()] - else: - res[field_name] = [x.value for x in rs.all()] - else: - x = rs.first() - if x: - if attr.value_type == 'datetime': - res[field_name] = datetime.datetime.strftime( - rs.first().value, '%Y-%m-%d %H:%M:%S') - else: - res[field_name] = rs.first().value - else: - res[field_name] = None - except AttributeError as e: - current_app.logger.warn("get ci by id error, {0}".format(e)) - if attr.is_multivalue: - res[field_name] = list() - else: - res[field_name] = "" - if uniq_key is not None and attr.attr_id == uniq_key.attr_id \ - and rs.first() is not None: - res['unique'] = uniq_key.attr_name - return res - - def _validate(self, attr, value, table, ci_id): - converter = type_map.get("converter").get(attr.value_type) - try: - v = converter(value) - except ValueError: - return False, "attribute value {0} converter fail".format(value) - if attr.is_choice: - choice_list = AttributeManager()._get_choice_value( - attr.attr_id, attr.value_type) - if v not in choice_list: - return False, "{0} is not existed in choice values".format( - value) - elif attr.is_uniq: - old_value = db.session.query(table.attr_id).filter( - table.attr_id == attr.attr_id).filter( - table.value == v).filter(table.ci_id != ci_id).first() - if old_value is not None: - return False, "attribute {0} value {1} must be unique".format( - attr.attr_name, value) - return True, v - - def add_attr_value(self, key, value, ci_id, ci_type, - _no_attribute_policy="ignore", ci_existed=False): - """key is one of attr_id, attr_name and attr_alias - """ - attr = self._get_attr(key) - if attr is None: - if _no_attribute_policy == 'ignore': - return True, None - if _no_attribute_policy == 'reject': - return False, 'attribute {0} not exist'.format(key) - table, old_value, old_value_table = TableMap( - attr_name=attr.attr_name).table, None, None - if ci_existed: - old_value_table = db.session.query(table).filter( - table.attr_id == attr.attr_id).filter( - table.ci_id == ci_id).first() - if old_value_table is not None: - old_value = old_value_table.value - if not value and ci_existed: - db.session.query(table).filter( - table.attr_id == attr.attr_id).filter( - table.ci_id == ci_id).delete() - if old_value: - return True, (attr.attr_id, "delete", old_value, None) - else: - return True, None - elif not value: - return True, None - if not attr.is_multivalue: - ret, res = self._validate(attr, value, table, ci_id) - if not ret: - return False, res - value_table = table() - if ci_existed: # for history - old = db.session.query(table).filter( - table.attr_id == attr.attr_id).filter( - table.value == value).filter( - table.ci_id == ci_id).first() - if old is not None: - return True, None - elif old_value_table: - value_table = old_value_table - value_table.ci_id = ci_id - value_table.attr_id = attr.attr_id - value_table.value = res - db.session.add(value_table) - elif attr.is_multivalue: - if ci_existed: - db.session.query(table).filter( - table.attr_id == attr.attr_id).filter( - table.ci_id == ci_id).delete() - - for v in value.strip().split(","): - ret, res = self._validate(attr, v, table, ci_id) - if not ret: - return False, res - value_table = table() - value_table.ci_id = ci_id - value_table.attr_id = attr.attr_id - value_table.value = res - db.session.add(value_table) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - current_app.logger.error( - "add attribute value is error, {0}".format(str(e))) - return False, "add attribute value is error, {0}".format(str(e)) - if ci_existed: - if old_value != value: - return True, (attr.attr_id, "update", old_value, value) - else: - return True, None - return True, (attr.attr_id, "add", None, value) diff --git a/manage.py b/manage.py deleted file mode 100644 index 3488926..0000000 --- a/manage.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- - - -from flask import jsonify -from flask import make_response -from flask.ext.script import Manager -from flask.ext.script import prompt_bool -from flask.ext.celery import install_commands as install_celery_command - -from __init__ import make_app -from extensions import db -from gunicornserver import GunicornServer -from lib.exception import InvalidUsageError - - -app = make_app('config.cfg') - - -@app.errorhandler(InvalidUsageError) -def handle_invalid_usage(error): - response = jsonify(error.to_dict()) - response.status_code = error.status_code - return response - - -@app.errorhandler(404) -def not_found(error): - return make_response(jsonify({'message': error.description}), 404) - - -@app.errorhandler(400) -def bad_request(error): - return make_response(jsonify({'message': error.description}), 400) - - -@app.errorhandler(401) -def auth_lack(error): - return make_response(jsonify({'message': error.description}), 401) - - -@app.errorhandler(403) -def exception_403(error): - return make_response(jsonify({'message': error.description}), 403) - - -@app.errorhandler(405) -def exception_405(error): - return make_response(jsonify({'message': error.description}), 405) - - -@app.errorhandler(500) -def server_error(error): - return make_response(jsonify({"message": error.description}), 500) - - -manager = Manager(app) - -install_celery_command(manager) - - -@manager.command -def db_setup(): - "create all database tables" - db.create_all() - - -@manager.command -def db_dropall(): - "drop all databse tables" - if prompt_bool("Are you sure ? You will lose all your data !"): - db.drop_all() - - -manager.add_command("run", GunicornServer()) - -if __name__ == '__main__': - manager.run(default_command="runserver") diff --git a/models/__init__.py b/models/__init__.py deleted file mode 100644 index f18317d..0000000 --- a/models/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding:utf-8 -*- - - -def row2dict(row): - d = dict() - for c in row.__table__.columns: - if not isinstance(getattr(row, c.name), - (basestring, long, int, float, list, tuple, dict)) \ - and getattr(row, c.name): - d[c.name] = getattr(row, c.name).strftime("%Y-%m-%d %H:%M:%S") - elif c.name not in ("password", "secret"): - d[c.name] = getattr(row, c.name) - return d \ No newline at end of file diff --git a/models/attribute.py b/models/attribute.py deleted file mode 100644 index 1f5081f..0000000 --- a/models/attribute.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding:utf-8 -*- - -from extensions import db, cache - - -class CIAttribute(db.Model): - __tablename__ = "ci_attributes" - - attr_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - attr_name = db.Column(db.String(32), nullable=False, unique=True) - attr_alias = db.Column(db.String(32), nullable=False, unique=True) - value_type = db.Column( - db.Enum("int", "float", "text", "datetime", name='value_type'), - default="text", - nullable=False) - is_choice = db.Column(db.Boolean, default=False) - is_multivalue = db.Column(db.Boolean, default=False) - is_uniq = db.Column(db.Boolean, default=False) - is_index = db.Column(db.Boolean, default=False) - - -class IntegerChoice(db.Model): - __tablename__ = 'choice_integers' - - choice_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - attr = db.relationship("CIAttribute", backref="choice_integers") - value = db.Column(db.Integer, nullable=False) - - -class FloatChoice(db.Model): - __tablename__ = 'choice_floats' - - choice_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - attr = db.relationship("CIAttribute", backref="choice_floats") - value = db.Column(db.Float, nullable=False) - - -class TextChoice(db.Model): - __tablename__ = 'choice_texts' - - choice_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - attr = db.relationship("CIAttribute", backref="choice_texts") - value = db.Column(db.Text, nullable=False) - - -class CIAttributeCache(object): - @classmethod - def get(cls, key): - if key is None: - return - attr = cache.get('Field::Name::%s' % key) or \ - cache.get('Field::ID::%s' % key) or \ - cache.get('Field::Alias::%s' % key) - if attr is None: - attr = db.session.query(CIAttribute).filter_by( - attr_name=key).first() or \ - db.session.query(CIAttribute).filter( - CIAttribute.attr_id == key).first() or \ - db.session.query(CIAttribute).filter( - CIAttribute.attr_alias == key).first() - db.session.close() - if attr is not None: - CIAttributeCache.set(attr) - return attr - - @classmethod - def set(cls, attr): - cache.set('Field::ID::%s' % attr.attr_id, attr) - cache.set('Field::Name::%s' % attr.attr_name, attr) - cache.set('Field::Alias::%s' % attr.attr_alias, attr) - - @classmethod - def clean(cls, attr): - if cache.get('Field::ID::%s' % attr.attr_id): - cache.delete('Field::ID::%s' % attr.attr_id) - cache.delete('Field::Name::%s' % attr.attr_name) - cache.delete('Field::Alias::%s' % attr.attr_alias) \ No newline at end of file diff --git a/models/ci.py b/models/ci.py deleted file mode 100644 index ab1a897..0000000 --- a/models/ci.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding:utf-8 -*- - -import datetime - -from extensions import db - - -class CI(db.Model): - __tablename__ = "cis" - - ci_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - uuid = db.Column(db.String(32), nullable=False) - type_id = db.Column(db.Integer, - db.ForeignKey("ci_types.type_id"), - nullable=False) - ci_type = db.relationship("CIType", backref="cis") - status = db.Column(db.Enum("review", "validate", name="status")) - created_time = db.Column(db.DateTime, default=datetime.datetime.now()) - heartbeat = db.Column(db.DateTime, default=datetime.datetime.now()) diff --git a/models/ci_relation.py b/models/ci_relation.py deleted file mode 100644 index 10585c9..0000000 --- a/models/ci_relation.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding:utf-8 -*- - - -from extensions import db - - -class CIRelation(db.Model): - __tablename__ = "ci_relations" - cr_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - first_ci_id = db.Column(db.Integer, - db.ForeignKey("cis.ci_id"), - primary_key=True) - second_ci_id = db.Column(db.Integer, - db.ForeignKey("cis.ci_id"), - primary_key=True) - first_ci = db.relationship("CI", - primaryjoin="CI.ci_id==CIRelation.first_ci_id") - second_ci = db.relationship( - "CI", primaryjoin="CI.ci_id==CIRelation.second_ci_id") - relation_type = db.Column( - db.Enum("connect", "deploy", "install", "contain", - name="relation_type"), nullable=False) - more = db.Column(db.Integer, db.ForeignKey("cis.ci_id")) - - __table_args__ = (db.UniqueConstraint("first_ci_id", "second_ci_id", - name="first_second_uniq"), ) diff --git a/models/ci_type.py b/models/ci_type.py deleted file mode 100644 index 7ec4e78..0000000 --- a/models/ci_type.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding:utf-8 -*- - -from extensions import db -from extensions import cache - - -class CIType(db.Model): - __tablename__ = "ci_types" - - type_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - type_name = db.Column(db.String(32)) - type_alias = db.Column(db.String(32)) - uniq_id = db.Column(db.Integer, - db.ForeignKey("ci_attributes.attr_id"), - nullable=False) - uniq_key = db.relationship("CIAttribute", backref="ci_types") - enabled = db.Column(db.Boolean, default=True, nullable=False) - is_attached = db.Column(db.Boolean, default=False, nullable=False) - icon_url = db.Column(db.String(256)) - order = db.Column(db.SmallInteger, default=0, nullable=False) - - -class CITypeAttribute(db.Model): - __tablename__ = "type_attributes" - - type_id = db.Column(db.Integer, - db.ForeignKey("ci_types.type_id"), - primary_key=True) - attr_id = db.Column(db.Integer, - db.ForeignKey("ci_attributes.attr_id"), - primary_key=True) - is_required = db.Column(db.Boolean, default=False) - order = db.Column(db.Integer, default=0) - - __table_args__ = (db.UniqueConstraint("type_id", "attr_id", - name="type_attr_uniq"), ) - - -class CITypeCache(object): - @classmethod - def get(cls, key): - if key is None: - return - ct = cache.get("CIType::ID::%s" % key) or \ - cache.get("CIType::Name::%s" % key) - if ct is None: - ct = db.session.query(CIType).filter( - CIType.type_name == key).first() or \ - db.session.query(CIType).filter(CIType.type_id == key).first() - if ct is not None: - CITypeCache.set(ct) - return ct - - @classmethod - def set(cls, ct): - cache.set("CIType::Name::%s" % ct.type_name, ct) - cache.set("CIType::ID::%d" % ct.type_id, ct) - - @classmethod - def clean(cls, key): - ct = CITypeCache.get(key) - if ct is not None: - cache.delete("CIType::Name::%s" % ct.type_name) - cache.delete("CIType::ID::%s" % ct.type_id) - - -class CITypeSpecCache(object): - @classmethod - def get(cls, key): - if key is None: - return - ct = cache.get("CITypeSPEC::ID::%d" % key) - if ct is None: - ct = db.session.query(CIType).filter(CIType.type_id == key).first() - if ct is not None: - CITypeSpecCache.set(ct) - return ct - - @classmethod - def set(cls, ct): - cache.set("CITypeSPEC::ID::%d" % ct.type_id, ct) - - @classmethod - def clean(cls, key): - ct = CITypeCache.get(key) - if ct is not None: - cache.delete("CITypeSPEC::ID::%d" % ct.type_id) - - -class CITypeAttributeCache(object): - """ - key is type_id or type_name - """ - - @classmethod - def get(cls, key): - if key is None: - return - if isinstance(key, basestring) and isinstance(key, unicode): - key = unicode(key, 'utf8') - citypes = cache.get("CITypeAttribute::Name::%s" % key) or \ - cache.get("CITypeAttribute::ID::%s" % key) - if not citypes: - citypes = db.session.query(CITypeAttribute).filter( - CITypeAttribute.type_id == key).all() - if citypes is None: - ci_type = db.session.query(CIType).filter( - CIType.type_name == key).first() - if ci_type is not None: - citypes = db.session.query(CITypeAttribute).filter_by( - type_id=ci_type.type_id).all() - if citypes is not None: - CITypeAttributeCache.set(key, citypes) - return citypes - - @classmethod - def set(cls, key, values): - citype = CITypeCache.get(key) - if citype is not None: - cache.set("CITypeAttribute::ID::%s" % citype.type_id, values) - cache.set("CITypeAttribute::Name::%s" % citype.type_name, values) - - @classmethod - def clean(cls, key): - citype = CITypeCache.get(key) - attrs = CITypeAttributeCache.get(key) - if attrs is not None and citype: - cache.delete("CITypeAttribute::ID::%s" % citype.type_id) - cache.delete("CITypeAttribute::Name::%s" % citype.type_name) \ No newline at end of file diff --git a/models/ci_type_relation.py b/models/ci_type_relation.py deleted file mode 100644 index b36509f..0000000 --- a/models/ci_type_relation.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding:utf-8 -*- - -from extensions import db - - -class CITypeRelation(db.Model): - __tablename__ = "ci_type_relations" - - ctr_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - parent_id = db.Column(db.Integer, - db.ForeignKey("ci_types.type_id"), - primary_key=True) - parent = db.relationship( - "CIType", primaryjoin="CIType.type_id==CITypeRelation.parent_id") - child_id = db.Column(db.Integer, - db.ForeignKey("ci_types.type_id"), - primary_key=True) - child = db.relationship( - "CIType", primaryjoin="CIType.type_id==CITypeRelation.child_id") - relation_type = db.Column( - db.Enum("contain", "connect", "deploy", "install", - name="relation_type"), - default="contain") - - __table_args__ = (db.UniqueConstraint("parent_id", "child_id", - name="parent_child_uniq"), ) \ No newline at end of file diff --git a/models/ci_value.py b/models/ci_value.py deleted file mode 100644 index f99068c..0000000 --- a/models/ci_value.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding:utf-8 -*- - - -from extensions import db -from sqlalchemy import Index - - -class CIIndexValueInteger(db.Model): - __tablename__ = "index_integers" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="index_integers") - attr = db.relationship("CIAttribute", backref="index_integers") - value = db.Column(db.Integer, nullable=False) - - __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) - - -class CIIndexValueFloat(db.Model): - __tablename__ = "index_floats" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="index_floats") - attr = db.relationship("CIAttribute", backref="index_floats") - value = db.Column(db.Float, nullable=False) - - __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) - - -class CIIndexValueText(db.Model): - __tablename__ = "index_texts" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="index_texts") - attr = db.relationship("CIAttribute", backref="index_texts") - value = db.Column(db.String(128), nullable=False) - - __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) - - -class CIIndexValueDateTime(db.Model): - __tablename__ = "index_datetime" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="index_datetime") - attr = db.relationship("CIAttribute", backref="index_datetime") - value = db.Column(db.DateTime, nullable=False) - - __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) - - -class CIValueInteger(db.Model): - __tablename__ = "integers" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="integers") - attr = db.relationship("CIAttribute", backref="integers") - value = db.Column(db.Integer, nullable=False) - - -class CIValueFloat(db.Model): - __tablename__ = "floats" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="floats") - attr = db.relationship("CIAttribute", backref="floats") - value = db.Column(db.Float, nullable=False) - - -class CIValueText(db.Model): - __tablename__ = "texts" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="texts") - attr = db.relationship("CIAttribute", backref="texts") - value = db.Column(db.Text, nullable=False) - - -class CIValueDateTime(db.Model): - __tablename__ = "datetime" - - value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) - attr_id = db.Column(db.Integer, - db.ForeignKey('ci_attributes.attr_id'), - nullable=False) - ci = db.relationship("CI", backref="datetime") - attr = db.relationship("CIAttribute", backref="datetime") - value = db.Column(db.DateTime, nullable=False) diff --git a/models/history.py b/models/history.py deleted file mode 100644 index e0bf10a..0000000 --- a/models/history.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from extensions import db - - -class OperationRecord(db.Model): - __tablename__ = "records" - - record_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - uid = db.Column(db.Integer, db.ForeignKey('users.uid'), nullable=False) - timestamp = db.Column(db.DateTime, - nullable=False, - default=datetime.datetime.now()) - origin = db.Column(db.String(32)) - ticket_id = db.Column(db.String(32)) - reason = db.Column(db.Text) - - -class CIAttributeHistory(db.Model): - __tablename__ = "histories" - - h_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - operate_type = db.Column(db.Enum("add", "delete", "update", - name="operate_type")) - record_id = db.Column(db.Integer, - db.ForeignKey("records.record_id"), - nullable=False) - ci_id = db.Column(db.Integer, nullable=False) - attr_id = db.Column(db.Integer, nullable=False) - old = db.Column(db.Text) - new = db.Column(db.Text) - - -class CIRelationHistory(db.Model): - __tablename__ = "relation_histories" - - rh_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - operate_type = db.Column(db.Enum("add", "delete", name="operate_type")) - record_id = db.Column(db.Integer, - db.ForeignKey("records.record_id"), - nullable=False) - first_ci_id = db.Column(db.Integer) - second_ci_id = db.Column(db.Integer) - relation_type = db.Column( - db.Enum("connect", "deploy", "install", "contain", - name="relation_type")) - relation = db.Column(db.Integer, nullable=False) diff --git a/models/statis.py b/models/statis.py deleted file mode 100644 index 64e03a4..0000000 --- a/models/statis.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from extensions import db - - -class UrlRecord(db.Model): - - url_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - - url = db.Column(db.String(64), nullable=False) - response_time = db.Column(db.Float, nullable=False) - is_ok = db.Column(db.Boolean, default=True) - source = db.Column(db.String(32)) - remote_addr = db.Column(db.String(20)) - hits = db.Column(db.Integer) - method = db.Column(db.String(5), default="GET") - created_at = db.Column(db.DateTime, default=datetime.datetime.now()) \ No newline at end of file diff --git a/permissions.py b/permissions.py deleted file mode 100644 index 891b952..0000000 --- a/permissions.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding:utf-8 -*- - - -from flask.ext.principal import RoleNeed, Permission - - -admin = Permission(RoleNeed('admin')) -auth = Permission(RoleNeed('authenticated')) -null = Permission(RoleNeed('null')) \ No newline at end of file diff --git a/requirements/default.txt b/requirements/default.txt deleted file mode 100644 index 6ee0a59..0000000 --- a/requirements/default.txt +++ /dev/null @@ -1,14 +0,0 @@ -Flask==0.9 -Flask-Script==0.5.2 -Flask-Babel==0.8 -Flask-principal==0.3.5 -Flask-mail==0.7.4 -pymysql==0.5 -sqlalchemy==0.8.2 -Flask-sqlalchemy==0.16 -Flask-cache==0.9.2 -redis==2.7.2 -gunicorn==0.17.4 -celery==3.0.18 -flask-celery=2.4.3 -Jinja2==2.7.1 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..97ac5d6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +[flake8] +ignore = D401,D202,E226,E302,E41 +max-line-length=120 +exclude = migrations/* +max-complexity = 10 + +[isort] +line_length=88 +multi_line_output=3 +skip=migrations/* +include_trailing_comma=true diff --git a/tasks/__init__.py b/tasks/__init__.py deleted file mode 100644 index 44d37d3..0000000 --- a/tasks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding:utf-8 -*- \ No newline at end of file diff --git a/tasks/statis.py b/tasks/statis.py deleted file mode 100644 index 0971117..0000000 --- a/tasks/statis.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding:utf-8 -*- - - -import datetime - -from flask import current_app - -from extensions import celery -from extensions import db -from models.statis import UrlRecord - - -@celery.task(name="statis.url_record", queue="statis_async") -def url_record(url, method, remote_addr, response_time, status_code, source): - current_app.logger.info("%s add 1" % url) - now = datetime.datetime.now() - u = UrlRecord(url=url, response_time=response_time, is_ok=1, - source="default", hits=1, method=method, created_at=now, - remote_addr=remote_addr) - db.session.add(u) - db.session.commit() \ No newline at end of file diff --git a/templates/search.xml b/templates/search.xml deleted file mode 100644 index e30f791..0000000 --- a/templates/search.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - {{ numfound }} - {{ page }} - - {% for ci in result %} - - {% for k, v in ci.items() %} - {% if not k.startswith('_') %} - {% for item in v | convert_to_list %} - {{ item }} - {% endfor %} - {% endif %} - {% endfor %} - - {% endfor %} - - - {% for k,v in facet.items() %} - - {% for item in v %} - {{ item[1] }} - {% endfor %} - - {% endfor %} - - \ No newline at end of file diff --git a/templates/search_tidy.xml b/templates/search_tidy.xml deleted file mode 100644 index 52832a0..0000000 --- a/templates/search_tidy.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - {{ code }} - - {% for k, v in ret.items() %} - - {% for ci in v %} - - - {% for item in ci|convert_to_list %} - {{ item }} - {% endfor %} - - - {% endfor %} - - {% endfor %} - - \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..b0c2ab8 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the app.""" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..003c8fc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +"""Defines fixtures available to all tests.""" + +import pytest +from webtest import TestApp + +from api.app import create_app + + +@pytest.fixture +def app(): + """Create application for the tests.""" + _app = create_app("tests.settings") + ctx = _app.test_request_context() + ctx.push() + yield _app + + ctx.pop() + + +@pytest.fixture +def testapp(app): + """Create Webtest app.""" + return TestApp(app) diff --git a/tests/test_cmdb_attribute.py b/tests/test_cmdb_attribute.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_attribute.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/test_cmdb_ci.py b/tests/test_cmdb_ci.py new file mode 100644 index 0000000..78a93f1 --- /dev/null +++ b/tests/test_cmdb_ci.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + + +class TestCI: + + def test_ci_search_only_type_query(self, app): + with app.test_client() as c: + rv = c.get('/api/v0.1/ci/s?q=_type:server', json={}) + json_data = rv.get_json() + assert type(json_data.get("result")) is list diff --git a/tests/test_cmdb_ci_realtion.py b/tests/test_cmdb_ci_realtion.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_ci_realtion.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/test_cmdb_ci_type.py b/tests/test_cmdb_ci_type.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_ci_type.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/test_cmdb_ci_type_relation.py b/tests/test_cmdb_ci_type_relation.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_ci_type_relation.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/test_cmdb_history.py b/tests/test_cmdb_history.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_history.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/test_cmdb_preference.py b/tests/test_cmdb_preference.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_preference.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/test_cmdb_relation_type.py b/tests/test_cmdb_relation_type.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/test_cmdb_relation_type.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*-