Initial commit

.gitignore

@@ -0,0 +1,74 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+!/.idea/
+secret.py
+

.idea/compiler.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <annotationProcessing>
+      <profile name="Maven default annotation processors profile" enabled="true">
+        <sourceOutputDir name="target/generated-sources/annotations" />
+        <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
+        <outputRelativeToContentRoot value="true" />
+        <module name="admin_panel" />
+      </profile>
+    </annotationProcessing>
+    <bytecodeTargetLevel>
+      <module name="admin_panel" target="1.8" />
+    </bytecodeTargetLevel>
+  </component>
+</project>

.idea/deployment.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" autoUpload="Always">
+    <option name="myAutoUpload" value="ALWAYS" />
+  </component>
+</project>

.idea/misc.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MavenProjectsManager">
+    <option name="originalFiles">
+      <list>
+        <option value="$PROJECT_DIR$/admin_panel/pom.xml" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="false" project-jdk-name="Python 3.8" project-jdk-type="Python SDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/KulYutmaz.iml" filepath="$PROJECT_DIR$/KulYutmaz.iml" />
+      <module fileurl="file://$PROJECT_DIR$/admin_panel/admin_panel.iml" filepath="$PROJECT_DIR$/admin_panel/admin_panel.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

AI.py

@@ -0,0 +1,7 @@
+import pickle
+import numpy
+
+def aiPredict(data):  # 32 lik liste yada deste alir.
+    with open("AIModel", "rb") as m:
+        aiModel = pickle.load(m)
+    return aiModel.predict(numpy.reshape(data,(1, 32)))    

AI/data_visualizer.py

@@ -0,0 +1,2 @@
+import pandas
+

AI/gather_data.py

@@ -0,0 +1,205 @@
+import pandas as pd
+import os
+import urllib.request
+from datetime import datetime
+from progressbar import ProgressBar
+import requests
+import yandex_search
+from urllib.parse import urlparse, quote
+import requests, json
+import urllib3
+from bs4 import BeautifulSoup
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+TRUSTED_TLDS = ["com", "org", "net"]
+pbar = None
+downloaded = 0
+
+
+def show_progress(count, block_size, total_size):
+	global pbar
+	global downloaded
+	if pbar is None:
+		pbar = ProgressBar(maxval=total_size)
+
+	downloaded += block_size
+	pbar.update(block_size)
+	if downloaded == total_size:
+		pbar.finish()
+		pbar = None
+		downloaded = 0
+
+
+def get_data(phishtank_key, force_update=False):
+	if not os.path.isfile("phishtank.csv") or force_update:
+		urllib.request.urlretrieve("http://data.phishtank.com/data/{}/online-valid.csv".format(phishtank_key),
+								   "phishtank.csv", show_progress)
+	if not os.path.isfile("common.csv") or force_update:
+		data = {"url":[]}
+		with open("keywordList") as wordlist:
+			keywords = wordlist.read().split("\n")
+			wordlist.close()
+		suggestions = []
+		for word in keywords:
+			URL = ("http://suggestqueries.google.com/complete/search?client=firefox&q="+word)
+			headers = {'User-agent':'Mozilla/5.0'}
+			response = requests.get(URL, headers=headers)
+			result = json.loads(response.content.decode('utf-8'))
+			for r in result[1]:
+				suggestions.append(r)
+		yandex = yandex_search.Yandex(api_user='yksiber', api_key='03.1041007756:28d93f7d79ff3c91b861da63e38a8e5c')
+		for word in suggestions:
+			top10 = (yandex.search(word).items[0:10])
+			for site in top10:
+				data["url"].append(site)
+		common = pd.DataFrame(data)
+		common.to_csv("common.csv")
+	urls = (pd.read_csv("phishtank.csv"), pd.read_csv("common.csv"))
+	return urls
+
+def find_list_resources (tag, attribute,soup):
+	list = []
+	for x in soup.findAll(tag):
+		try:
+			list.append(x[attribute])
+		except KeyError:
+			pass
+	return(list)
+
+def get_url_data(url,yandex,timeout=30):
+	#Basic data extraction
+	data = {}
+	data["length"] = (len(url.split("://")[1].split("?")[0]))
+	data["dir_num"] = (url.find("/")-2)
+	parsed = urlparse(url)
+	hostname_split = parsed.hostname.split(".")
+	data["tld_trust"] = int(hostname_split[-1].lower() in TRUSTED_TLDS)
+	data["subdomain_num"] = len(hostname_split) - 2
+	data["subdomain_len"] = len("".join(hostname_split[:-2]))
+	special_char_count = 0
+	for char in parsed.hostname:
+		if char == ".":
+			continue
+		if not char.encode("utf-8") == char.encode("idna"):
+			special_char_count += 1
+	data["special_char_num"] = special_char_count
+	#Advanced data extraction
+	try:
+		data["index_num"] = yandex.search("site:{}".format(parsed.hostname)).found["all"]
+	except yandex_search.NoResultsException:
+		data["index_num"] = 0
+	robot_entry_counter = 0
+	try:
+		response = requests.get("{}://{}/robots.txt".format(parsed.scheme, parsed.netloc), allow_redirects=True, verify=False, timeout=timeout)
+		if response.status_code == 200:
+			lines = response.text.split("\n")
+			lines = [x for x in lines if x != ""]
+			robot_entry_counter += len([x for x in lines if x[0] != "#"])
+		else:
+			pass
+	except Exception as e:
+		print(e)
+	data["robots_entries"] = robot_entry_counter
+	try:
+		req = requests.get(url, verify=False, timeout=timeout)
+		if req.status_code == 200:
+			soup = BeautifulSoup(req.text,'html.parser')
+			image_scr = find_list_resources('img',"src",soup)
+			script_src = find_list_resources('script',"src",soup)
+			css_link = find_list_resources("link","href",soup)
+			all_links = image_scr + css_link + script_src
+			out_links = []
+			for link in all_links:
+				parsed_link = urlparse(link)
+				if parsed_link.hostname != parsed.hostname:
+					out_links.append(link)
+			data["out_resources"] = len(out_links)
+		else:
+			data["out_resources"] = -1
+	except Exception as e:
+		print(e)
+		data["out_resources"] = -1
+	data["url"] = url
+	return data
+
+
+def extract_data(raw_data, force_update=False):
+	reps = 0
+	phishing, benign = raw_data[0], raw_data[1]
+	data = {
+		"phishing": [],
+		"length": [],
+		"out_resources": [],
+		"dir_num": [],
+		"special_char_num": [],
+		"robots_entries": [],
+		"tld_trust": [],
+		"index_num": [],
+		"subdomain_len": [],
+		"subdomain_num": [],
+		"url": []
+	}
+	if not os.path.isfile("dataset.csv") or force_update:
+		largest_dataset = 0
+		while os.path.isfile(largest_dataset + 300):
+			largest_dataset += 300
+		try:
+			# filter old sites
+			old = []
+			for index, row in phishing.iterrows():
+				date = datetime.strptime(row["submission_time"],"%Y-%m-%dT%H:%M:%S+00:00")
+				if date.year < 2020:
+					old.append(index)
+			phishing = phishing.drop(old)
+			yandex = yandex_search.Yandex(api_user='yksiber', api_key='03.1041007756:28d93f7d79ff3c91b861da63e38a8e5c')
+			for index, row in phishing.iterrows():
+				reps += 1
+				if reps < largest_dataset:
+					continue
+				if reps % 300 == 0:
+					pd.DataFrame(data).to_csv("dataset{}.csv".format(reps))
+				url = row['url']
+				print("[INFO]: {} : {}".format(reps, url))
+				url_data = get_url_data(url, yandex)
+				data["phishing"].append(1)
+				data["length"].append(url_data["length"])
+				data["dir_num"].append(url_data["dir_num"])
+				data["special_char_num"].append(url_data["special_char_num"])
+				data["tld_trust"].append(url_data["tld_trust"])
+				data["index_num"].append(url_data["index_num"])
+				data["subdomain_len"].append(url_data["subdomain_len"])
+				data["subdomain_num"].append(url_data["subdomain_num"])
+				data["out_resources"].append(url_data["out_resources"])
+				data["robots_entries"].append(url_data["robots_entries"])
+				data["url"].append(url_data["url"])
+			for index, row in benign.iterrows():
+				reps += 1
+				if reps < largest_dataset:
+					continue
+				if reps % 300 == 0:
+					pd.DataFrame(data).to_csv("dataset{}.csv".format(reps))
+				url = row['url']
+				print("[INFO]: {} : {}".format(reps, url))
+				url_data = get_url_data(url, yandex)
+				data["phishing"].append(1)
+				data["length"].append(url_data["length"])
+				data["dir_num"].append(url_data["dir_num"])
+				data["special_char_num"].append(url_data["special_char_num"])
+				data["tld_trust"].append(url_data["tld_trust"])
+				data["index_num"].append(url_data["index_num"])
+				data["subdomain_len"].append(url_data["subdomain_len"])
+				data["subdomain_num"].append(url_data["subdomain_num"])
+				data["out_resources"].append(url_data["out_resources"])
+				data["robots_entries"].append(url_data["robots_entries"])
+				data["url"].append(url_data["url"])
+			pd.DataFrame(data).to_csv("dataset.csv".format(reps))
+		except Exception as e:
+			print("[ERROR]: {}".format(e))
+	return pd.read_csv("dataset.csv")
+
+
+raw_data = get_data("01115eebdbf465734c08fedb2e4d93f414d1a31fa10bfcb248d0f75071e156ff")
+print("DOWNLOAD COMPLETED!")
+dataset = extract_data(raw_data)
+print("EXTRACT COMPLETED!")

DatabaseUpdater.py

@@ -0,0 +1,6 @@
+import mysql.connector
+
+kulyutmazDatabase = ""
+
+cloudDatabase = mysql.connector.connect(kulyutmazDatabase)
+localDatabase = mysql.connector.connect(host)

KulYutmaz.iml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="FacetManager">
+    <facet type="Python" name="Python">
+      <configuration sdkName="" />
+    </facet>
+  </component>
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

README.md

@@ -0,0 +1,7 @@
+# KulYutmaz
+
+<p align="center">
+  <img width="128" height="128" src="/spam.png">
+</p>
+
+KulYutmaz is a project developed for the regional TUBITAK competition's programming field. It aims to create a more advanced phishing e-mail detection algorithm using website content checking and a neural network that we have trained. For more information, you may read the project report in the repository (Unfortunately it is only available in Turkish). 

XSSfinder.py

@@ -0,0 +1,9 @@
+suspiciusLetters = ["<",">","|",";","%"] # Buraya birkaç karakter daha eklemek gerekebilir.
+
+def XSSsearcher(url): # Bu algoritma kolay olsada iyi çalışıyor gibi duruyor. (DOM için çalışıyor sadece)
+    for letter in url:
+        if letter in suspiciusLetters:
+            return True
+    return False
+
+

admin_panel/admin_panel.iml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="jdk" jdkName="1.8" jdkType="JavaSDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module-library">
+      <library>
+        <CLASSES>
+          <root url="jar://$USER_HOME$/Downloads/json-20190722.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES />
+      </library>
+    </orderEntry>
+    <orderEntry type="library" name="Maven: org.jfree:jfreechart:1.5.0" level="project" />
+    <orderEntry type="library" name="Maven: mysql:mysql-connector-java:8.0.18" level="project" />
+    <orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:3.6.1" level="project" />
+    <orderEntry type="library" name="Maven: joda-time:joda-time:2.10.5" level="project" />
+    <orderEntry type="library" name="Maven: org.json:json:20190722" level="project" />
+  </component>
+</module>

admin_panel/nb-configuration.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project-shared-configuration>
+    <!--
+This file contains additional configuration written by modules in the NetBeans IDE.
+The configuration is intended to be shared among all the users of project and
+therefore it is assumed to be part of version control checkout.
+Without this configuration present, some functionality in the IDE may be limited or fail altogether.
+-->
+    <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
+        <!--
+Properties that influence various parts of the IDE, especially code formatting and the like. 
+You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
+That way multiple projects can share the same settings (useful for formatting rules for example).
+Any value defined here will override the pom.xml file value but is only applicable to the current project.
+-->
+        <netbeans.hint.jdkPlatform>JDK_11</netbeans.hint.jdkPlatform>
+    </properties>
+</project-shared-configuration>

admin_panel/nbactions.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+        <action>
+            <actionName>run</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.5.0:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath com.yigitcolakoglu.kulyutmaz.Control_Panel</exec.args>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+        <action>
+            <actionName>debug</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.5.0:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath com.yigitcolakoglu.kulyutmaz.Control_Panel</exec.args>
+                <exec.executable>java</exec.executable>
+                <jpda.listen>true</jpda.listen>
+            </properties>
+        </action>
+        <action>
+            <actionName>profile</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.5.0:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath com.yigitcolakoglu.kulyutmaz.Control_Panel</exec.args>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+    </actions>

admin_panel/pom.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+		 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>org.example</groupId>
+	<artifactId>admin_panel</artifactId>
+	<version>1.0-SNAPSHOT</version>
+    <repositories>
+
+		<repository>
+			<id>netbeans</id>
+			<name>Netbeans rep</name>
+			<url>https://repo1.maven.org/maven2/</url>
+		</repository>
+		<repository>
+			<id>central</id>
+			<name>central rep</name>
+			<url>http://bits.netbeans.org/maven2/</url>
+		</repository>
+	</repositories>
+    <dependencies>
+		<dependency>
+			<groupId>org.jfree</groupId>
+			<artifactId>jfreechart</artifactId>
+			<version>1.5.0</version>
+		</dependency>
+		<dependency>
+			<groupId>mysql</groupId>1
+			<artifactId>mysql-connector-java</artifactId>
+			<version>8.0.18</version>
+		</dependency>
+		<dependency>
+			<groupId>joda-time</groupId>
+			<artifactId>joda-time</artifactId>
+			<version>2.10.5</version>
+		</dependency>
+		<!-- https://mvnrepository.com/artifact/org.json/json -->
+		<dependency>
+			<groupId>org.json</groupId>
+			<artifactId>json</artifactId>
+			<version>20190722</version>
+		</dependency>
+
+	</dependencies>
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+</project>